{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cd24ebe8",
   "metadata": {},
   "source": [
    "# 模型boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "38068292",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "# dnn模型构建\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
    "import torch.nn.functional as F\n",
    "import random\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "        datefmt='%m/%d/%Y %H:%M:%S',\n",
    "        level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "feea2fd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 固定随机数种子，确保实验的可重复性\n",
    "def set_seed(seed=42):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "set_seed(42)\n",
    "\n",
    "def get_fft_and_scaler(data, start=5192, end=8192):\n",
    "    data = np.fft.fft(data)\n",
    "    data = np.abs(data)\n",
    "    data = data/np.expand_dims(data.max(axis=1), axis=1)\n",
    "    return data[:, start:end]\n",
    "\n",
    "# 搭建DNN模型\n",
    "class DNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.dnn = nn.Sequential(\n",
    "            nn.BatchNorm1d(300),\n",
    "            nn.Linear(300, 1024),\n",
    "            nn.BatchNorm1d(1024),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.2),\n",
    "            nn.Linear(1024, 256),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.2),\n",
    "            nn.Linear(256, 10),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.dnn(x)\n",
    "        return F.softmax(x, dim=1)\n",
    "\n",
    "def get_boost_data(feature, preds_label, true_label):\n",
    "    return feature[preds_label == true_label, :].detach().cpu(), true_label[preds_label == true_label].detach().cpu()\n",
    "\n",
    "def model_train(train_loader, model, optimizer, criterion, labels, boost=True):\n",
    "    model.train()\n",
    "    train_total_acc = 0\n",
    "    train_loss = 0\n",
    "    for feature, label in train_loader:\n",
    "        feature = feature.to(device)\n",
    "        label = label.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        preds = model(feature)\n",
    "\n",
    "        loss = criterion(preds, label)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        # 存储boosting数据\n",
    "        if epoch >= epochs - boost_epoch_num  and boost:\n",
    "            d1, d2 = get_boost_data(feature, preds.argmax(dim=1), label)\n",
    "            boost_feature.append(d1)\n",
    "            boost_label.append(d2)\n",
    "\n",
    "        train_total_acc += model(feature).argmax(dim=1).eq(label).sum().item()\n",
    "        train_loss += loss.item()\n",
    "\n",
    "        feature.cpu()\n",
    "        label.cpu()\n",
    "\n",
    "    print(\n",
    "        f'Training loss: {train_loss/len(train_loader):.4f}',\n",
    "        f'Training  acc: {train_total_acc/len(labels):.4f}',\n",
    "         )\n",
    "\n",
    "def predict(val_loader, model, criterion, labels):\n",
    "    model.eval()\n",
    "    val_total_acc = 0\n",
    "    val_loss = 0\n",
    "    for feature, label in val_loader:\n",
    "        feature = feature.to(device)\n",
    "        label = label.to(device)\n",
    "        preds = model(feature)\n",
    "        loss = criterion(preds, label)\n",
    "\n",
    "        val_total_acc += model(feature).argmax(dim=1).eq(label).sum().item()\n",
    "        val_loss += loss.item()\n",
    "\n",
    "        feature.cpu()\n",
    "        label.cpu()\n",
    "\n",
    "    print(\n",
    "        f'Val loss: {val_loss/len(val_loader):.4f}',\n",
    "        f'Val  acc:{val_total_acc/len(labels):.4f}'\n",
    "    )\n",
    "    return val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6577bc89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading data\n"
     ]
    }
   ],
   "source": [
    "print('loading data')\n",
    "# 读取训练集，测试集和验证集\n",
    "train = np.load('../train/10type_sort_train_data_8192.npy')\n",
    "val = np.load('../val/10type_sort_eval_data_8192.npy')\n",
    "\n",
    "# 读取训练集和验证集的标签，测试集是没有标签的，需要你使用模型进行分类，并将结果进行提交\n",
    "train_label = np.load('../train/10type_sort_train_label_8192.npy')\n",
    "val_label = np.load('../val/10type_sort_eval_label_8192.npy')\n",
    "\n",
    "train_sp = get_fft_and_scaler(train, start=6892, end=7192)\n",
    "val_sp = get_fft_and_scaler(val, start=6892, end=7192)\n",
    "\n",
    "# 将数据转换成pytorch的tensor\n",
    "batch_size = 128\n",
    "\n",
    "train_tensor = torch.tensor(train_sp).float()\n",
    "y_train_tensor = torch.tensor(train_label).long()\n",
    "val_tensor = torch.tensor(val_sp).float()\n",
    "y_val_tensor = torch.tensor(val_label).long()\n",
    "\n",
    "# 使用Dataloader对数据进行封装\n",
    "val_tensor = TensorDataset(val_tensor, y_val_tensor)\n",
    "val_loader = DataLoader(val_tensor, shuffle=False, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f9f8fdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.0001\n",
    "gamma = 0.9\n",
    "step_size = 1\n",
    "epochs = 30\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "247dc361",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------boost round: 0/4----------------\n",
      "==================== Epoch: 0 ====================\n",
      "Training loss: 2.0387 Training  acc: 0.4588\n",
      "Val loss: 2.0017 Val  acc:0.4544\n",
      "==================== Epoch: 1 ====================\n",
      "Training loss: 1.9170 Training  acc: 0.5610\n",
      "Val loss: 1.9623 Val  acc:0.5002\n",
      "==================== Epoch: 2 ====================\n",
      "Training loss: 1.8857 Training  acc: 0.5879\n",
      "Val loss: 1.9482 Val  acc:0.5132\n",
      "==================== Epoch: 3 ====================\n",
      "Training loss: 1.8708 Training  acc: 0.6010\n",
      "Val loss: 1.9466 Val  acc:0.5114\n",
      "==================== Epoch: 4 ====================\n",
      "Training loss: 1.8485 Training  acc: 0.6259\n",
      "Val loss: 1.9138 Val  acc:0.5527\n",
      "==================== Epoch: 5 ====================\n",
      "Training loss: 1.8342 Training  acc: 0.6397\n",
      "Val loss: 1.9162 Val  acc:0.5462\n",
      "==================== Epoch: 6 ====================\n",
      "Training loss: 1.8256 Training  acc: 0.6471\n",
      "Val loss: 1.9051 Val  acc:0.5580\n",
      "==================== Epoch: 7 ====================\n",
      "Training loss: 1.8194 Training  acc: 0.6517\n",
      "Val loss: 1.8970 Val  acc:0.5663\n",
      "==================== Epoch: 8 ====================\n",
      "Training loss: 1.8123 Training  acc: 0.6582\n",
      "Val loss: 1.8909 Val  acc:0.5740\n",
      "==================== Epoch: 9 ====================\n",
      "Training loss: 1.8074 Training  acc: 0.6621\n",
      "Val loss: 1.8977 Val  acc:0.5652\n",
      "==================== Epoch: 10 ====================\n",
      "Training loss: 1.8032 Training  acc: 0.6658\n",
      "Val loss: 1.8961 Val  acc:0.5660\n",
      "==================== Epoch: 11 ====================\n",
      "Training loss: 1.7978 Training  acc: 0.6728\n",
      "Val loss: 1.8924 Val  acc:0.5705\n",
      "==================== Epoch: 12 ====================\n",
      "Training loss: 1.7941 Training  acc: 0.6755\n",
      "Val loss: 1.8828 Val  acc:0.5815\n",
      "==================== Epoch: 13 ====================\n",
      "Training loss: 1.7918 Training  acc: 0.6785\n",
      "Val loss: 1.8905 Val  acc:0.5724\n",
      "==================== Epoch: 14 ====================\n",
      "Training loss: 1.7876 Training  acc: 0.6819\n",
      "Val loss: 1.8851 Val  acc:0.5770\n",
      "==================== Epoch: 15 ====================\n",
      "Training loss: 1.7843 Training  acc: 0.6846\n",
      "Val loss: 1.8805 Val  acc:0.5848\n",
      "==================== Epoch: 16 ====================\n",
      "Training loss: 1.7807 Training  acc: 0.6872\n",
      "Val loss: 1.8786 Val  acc:0.5830\n",
      "==================== Epoch: 17 ====================\n",
      "Training loss: 1.7787 Training  acc: 0.6886\n",
      "Val loss: 1.8760 Val  acc:0.5866\n",
      "==================== Epoch: 18 ====================\n",
      "Training loss: 1.7754 Training  acc: 0.6925\n",
      "Val loss: 1.8842 Val  acc:0.5768\n",
      "==================== Epoch: 19 ====================\n",
      "Training loss: 1.7727 Training  acc: 0.6953\n",
      "Val loss: 1.8884 Val  acc:0.5704\n",
      "==================== Epoch: 20 ====================\n",
      "Training loss: 1.7696 Training  acc: 0.6986\n",
      "Val loss: 1.8862 Val  acc:0.5749\n",
      "==================== Epoch: 21 ====================\n",
      "Training loss: 1.7689 Training  acc: 0.6982\n",
      "Val loss: 1.8805 Val  acc:0.5805\n",
      "==================== Epoch: 22 ====================\n",
      "Training loss: 1.7667 Training  acc: 0.7012\n",
      "Val loss: 1.8794 Val  acc:0.5809\n",
      "==================== Epoch: 23 ====================\n",
      "Training loss: 1.7647 Training  acc: 0.7021\n",
      "Val loss: 1.8795 Val  acc:0.5811\n",
      "==================== Epoch: 24 ====================\n",
      "Training loss: 1.7621 Training  acc: 0.7050\n",
      "Val loss: 1.8819 Val  acc:0.5791\n",
      "==================== Epoch: 25 ====================\n",
      "Training loss: 1.7616 Training  acc: 0.7062\n",
      "Val loss: 1.8810 Val  acc:0.5803\n",
      "==================== Epoch: 26 ====================\n",
      "Training loss: 1.7596 Training  acc: 0.7092\n",
      "Val loss: 1.8758 Val  acc:0.5843\n",
      "==================== Epoch: 27 ====================\n",
      "Training loss: 1.7575 Training  acc: 0.7087\n",
      "Val loss: 1.8767 Val  acc:0.5839\n",
      "==================== Epoch: 28 ====================\n",
      "Training loss: 1.7565 Training  acc: 0.7103\n",
      "Val loss: 1.8746 Val  acc:0.5848\n",
      "==================== Epoch: 29 ====================\n",
      "Training loss: 1.7547 Training  acc: 0.7124\n",
      "Val loss: 1.8811 Val  acc:0.5782\n",
      "----------------boost round: 1/4----------------\n",
      "==================== Epoch: 0 ====================\n",
      "Training loss: 1.7528 Training  acc: 0.7513\n",
      "Val loss: 1.9307 Val  acc:0.5332\n",
      "==================== Epoch: 1 ====================\n",
      "Training loss: 1.6294 Training  acc: 0.8479\n",
      "Val loss: 1.9177 Val  acc:0.5405\n",
      "==================== Epoch: 2 ====================\n",
      "Training loss: 1.5930 Training  acc: 0.8796\n",
      "Val loss: 1.9084 Val  acc:0.5495\n",
      "==================== Epoch: 3 ====================\n",
      "Training loss: 1.5712 Training  acc: 0.9027\n",
      "Val loss: 1.8841 Val  acc:0.5773\n",
      "==================== Epoch: 4 ====================\n",
      "Training loss: 1.5492 Training  acc: 0.9247\n",
      "Val loss: 1.8772 Val  acc:0.5849\n",
      "==================== Epoch: 5 ====================\n",
      "Training loss: 1.5341 Training  acc: 0.9384\n",
      "Val loss: 1.8669 Val  acc:0.5956\n",
      "==================== Epoch: 6 ====================\n",
      "Training loss: 1.5241 Training  acc: 0.9475\n",
      "Val loss: 1.8728 Val  acc:0.5868\n",
      "==================== Epoch: 7 ====================\n",
      "Training loss: 1.5175 Training  acc: 0.9531\n",
      "Val loss: 1.8719 Val  acc:0.5886\n",
      "==================== Epoch: 8 ====================\n",
      "Training loss: 1.5127 Training  acc: 0.9568\n",
      "Val loss: 1.8761 Val  acc:0.5841\n",
      "==================== Epoch: 9 ====================\n",
      "Training loss: 1.5082 Training  acc: 0.9600\n",
      "Val loss: 1.8726 Val  acc:0.5861\n",
      "==================== Epoch: 10 ====================\n",
      "Training loss: 1.5058 Training  acc: 0.9626\n",
      "Val loss: 1.8740 Val  acc:0.5857\n",
      "==================== Epoch: 11 ====================\n",
      "Training loss: 1.5026 Training  acc: 0.9648\n",
      "Val loss: 1.8680 Val  acc:0.5922\n",
      "==================== Epoch: 12 ====================\n",
      "Training loss: 1.5002 Training  acc: 0.9673\n",
      "Val loss: 1.8667 Val  acc:0.5932\n",
      "==================== Epoch: 13 ====================\n",
      "Training loss: 1.4980 Training  acc: 0.9692\n",
      "Val loss: 1.8727 Val  acc:0.5865\n",
      "==================== Epoch: 14 ====================\n",
      "Training loss: 1.4967 Training  acc: 0.9697\n",
      "Val loss: 1.8713 Val  acc:0.5874\n",
      "==================== Epoch: 15 ====================\n",
      "Training loss: 1.4947 Training  acc: 0.9715\n",
      "Val loss: 1.8730 Val  acc:0.5862\n",
      "==================== Epoch: 16 ====================\n",
      "Training loss: 1.4931 Training  acc: 0.9726\n",
      "Val loss: 1.8734 Val  acc:0.5853\n",
      "==================== Epoch: 17 ====================\n",
      "Training loss: 1.4916 Training  acc: 0.9737\n",
      "Val loss: 1.8707 Val  acc:0.5882\n",
      "==================== Epoch: 18 ====================\n",
      "Training loss: 1.4909 Training  acc: 0.9746\n",
      "Val loss: 1.8756 Val  acc:0.5840\n",
      "==================== Epoch: 19 ====================\n",
      "Training loss: 1.4901 Training  acc: 0.9750\n",
      "Val loss: 1.8646 Val  acc:0.5946\n",
      "==================== Epoch: 20 ====================\n",
      "Training loss: 1.4882 Training  acc: 0.9769\n",
      "Val loss: 1.8743 Val  acc:0.5843\n",
      "==================== Epoch: 21 ====================\n",
      "Training loss: 1.4873 Training  acc: 0.9774\n",
      "Val loss: 1.8700 Val  acc:0.5890\n",
      "==================== Epoch: 22 ====================\n",
      "Training loss: 1.4871 Training  acc: 0.9780\n",
      "Val loss: 1.8697 Val  acc:0.5896\n",
      "==================== Epoch: 23 ====================\n",
      "Training loss: 1.4865 Training  acc: 0.9786\n",
      "Val loss: 1.8651 Val  acc:0.5937\n",
      "==================== Epoch: 24 ====================\n",
      "Training loss: 1.4850 Training  acc: 0.9797\n",
      "Val loss: 1.8739 Val  acc:0.5851\n",
      "==================== Epoch: 25 ====================\n",
      "Training loss: 1.4844 Training  acc: 0.9795\n",
      "Val loss: 1.8786 Val  acc:0.5794\n",
      "==================== Epoch: 26 ====================\n",
      "Training loss: 1.4843 Training  acc: 0.9798\n",
      "Val loss: 1.8706 Val  acc:0.5883\n",
      "==================== Epoch: 27 ====================\n",
      "Training loss: 1.4830 Training  acc: 0.9809\n",
      "Val loss: 1.8703 Val  acc:0.5892\n",
      "==================== Epoch: 28 ====================\n",
      "Training loss: 1.4834 Training  acc: 0.9808\n",
      "Val loss: 1.8683 Val  acc:0.5907\n",
      "==================== Epoch: 29 ====================\n",
      "Training loss: 1.4822 Training  acc: 0.9815\n",
      "Val loss: 1.8754 Val  acc:0.5830\n",
      "----------------boost round: 2/4----------------\n",
      "==================== Epoch: 0 ====================\n",
      "Training loss: 1.6260 Training  acc: 0.8550\n",
      "Val loss: 1.8748 Val  acc:0.5892\n",
      "==================== Epoch: 1 ====================\n",
      "Training loss: 1.5239 Training  acc: 0.9488\n",
      "Val loss: 1.8718 Val  acc:0.5888\n",
      "==================== Epoch: 2 ====================\n",
      "Training loss: 1.4990 Training  acc: 0.9693\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val loss: 1.8716 Val  acc:0.5878\n",
      "==================== Epoch: 3 ====================\n",
      "Training loss: 1.4893 Training  acc: 0.9772\n",
      "Val loss: 1.8683 Val  acc:0.5909\n",
      "==================== Epoch: 4 ====================\n",
      "Training loss: 1.4839 Training  acc: 0.9815\n",
      "Val loss: 1.8757 Val  acc:0.5833\n",
      "==================== Epoch: 5 ====================\n",
      "Training loss: 1.4806 Training  acc: 0.9842\n",
      "Val loss: 1.8717 Val  acc:0.5874\n",
      "==================== Epoch: 6 ====================\n",
      "Training loss: 1.4778 Training  acc: 0.9864\n",
      "Val loss: 1.8621 Val  acc:0.5970\n",
      "==================== Epoch: 7 ====================\n",
      "Training loss: 1.4761 Training  acc: 0.9877\n",
      "Val loss: 1.8665 Val  acc:0.5926\n",
      "==================== Epoch: 8 ====================\n",
      "Training loss: 1.4747 Training  acc: 0.9887\n",
      "Val loss: 1.8679 Val  acc:0.5909\n",
      "==================== Epoch: 9 ====================\n",
      "Training loss: 1.4737 Training  acc: 0.9897\n",
      "Val loss: 1.8677 Val  acc:0.5916\n",
      "==================== Epoch: 10 ====================\n",
      "Training loss: 1.4728 Training  acc: 0.9904\n",
      "Val loss: 1.8731 Val  acc:0.5856\n",
      "==================== Epoch: 11 ====================\n",
      "Training loss: 1.4722 Training  acc: 0.9909\n",
      "Val loss: 1.8719 Val  acc:0.5866\n",
      "==================== Epoch: 12 ====================\n",
      "Training loss: 1.4712 Training  acc: 0.9916\n",
      "Val loss: 1.8655 Val  acc:0.5939\n",
      "==================== Epoch: 13 ====================\n",
      "Training loss: 1.4707 Training  acc: 0.9921\n",
      "Val loss: 1.8713 Val  acc:0.5877\n",
      "==================== Epoch: 14 ====================\n",
      "Training loss: 1.4703 Training  acc: 0.9925\n",
      "Val loss: 1.8746 Val  acc:0.5851\n",
      "==================== Epoch: 15 ====================\n",
      "Training loss: 1.4698 Training  acc: 0.9928\n",
      "Val loss: 1.8700 Val  acc:0.5895\n",
      "==================== Epoch: 16 ====================\n",
      "Training loss: 1.4695 Training  acc: 0.9931\n",
      "Val loss: 1.8676 Val  acc:0.5916\n",
      "==================== Epoch: 17 ====================\n",
      "Training loss: 1.4690 Training  acc: 0.9934\n",
      "Val loss: 1.8710 Val  acc:0.5891\n",
      "==================== Epoch: 18 ====================\n",
      "Training loss: 1.4689 Training  acc: 0.9935\n",
      "Val loss: 1.8732 Val  acc:0.5864\n",
      "==================== Epoch: 19 ====================\n",
      "Training loss: 1.4685 Training  acc: 0.9938\n",
      "Val loss: 1.8655 Val  acc:0.5942\n",
      "==================== Epoch: 20 ====================\n",
      "Training loss: 1.4682 Training  acc: 0.9940\n",
      "Val loss: 1.8746 Val  acc:0.5845\n",
      "==================== Epoch: 21 ====================\n",
      "Training loss: 1.4681 Training  acc: 0.9942\n",
      "Val loss: 1.8712 Val  acc:0.5877\n",
      "==================== Epoch: 22 ====================\n",
      "Training loss: 1.4677 Training  acc: 0.9944\n",
      "Val loss: 1.8675 Val  acc:0.5917\n",
      "==================== Epoch: 23 ====================\n",
      "Training loss: 1.4675 Training  acc: 0.9946\n",
      "Val loss: 1.8658 Val  acc:0.5931\n",
      "==================== Epoch: 24 ====================\n",
      "Training loss: 1.4673 Training  acc: 0.9950\n",
      "Val loss: 1.8681 Val  acc:0.5918\n",
      "==================== Epoch: 25 ====================\n",
      "Training loss: 1.4671 Training  acc: 0.9950\n",
      "Val loss: 1.8679 Val  acc:0.5916\n",
      "==================== Epoch: 26 ====================\n",
      "Training loss: 1.4671 Training  acc: 0.9948\n",
      "Val loss: 1.8679 Val  acc:0.5912\n",
      "==================== Epoch: 27 ====================\n",
      "Training loss: 1.4670 Training  acc: 0.9951\n",
      "Val loss: 1.8699 Val  acc:0.5892\n",
      "==================== Epoch: 28 ====================\n",
      "Training loss: 1.4667 Training  acc: 0.9952\n",
      "Val loss: 1.8710 Val  acc:0.5879\n",
      "==================== Epoch: 29 ====================\n",
      "Training loss: 1.4666 Training  acc: 0.9953\n",
      "Val loss: 1.8602 Val  acc:0.5992\n",
      "----------------boost round: 3/4----------------\n",
      "==================== Epoch: 0 ====================\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-22a96f221d25>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'='\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m20\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34mf' Epoch: {epoch} '\u001b[0m\u001b[0;34m+\u001b[0m \u001b[0;34m'='\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m         \u001b[0mmodel_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_train_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mboost\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_val_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mtrain_best\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-7fac72fa817c>\u001b[0m in \u001b[0;36mmodel_train\u001b[0;34m(train_loader, model, optimizer, criterion, labels, boost)\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[0mtrain_total_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mfeature\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m         \u001b[0mfeature\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[0;34m(batch)\u001b[0m\n\u001b[1;32m     81\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     81\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[0;34m(batch)\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mstorage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_new_shared\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__module__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'numpy'\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'str_'\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'string_'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 使用boost ing的想法，让神经网络学习错误的类别\n",
    "# boosting 的想法：训练后面几次，分类错误的数据\n",
    "# 重新定义一个新的分类器进行学习错误的数据, 保存这些模型的参数\n",
    "# 然后使用模型融合\n",
    "boost_epoch_num = 4\n",
    "for boost_num in range(boost_epoch_num):\n",
    "    # 分类器学习, 更新训练集\n",
    "    train_dataset = TensorDataset(train_tensor, y_train_tensor)\n",
    "    train_loader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "    model = DNN().to(device)\n",
    "    boost_feature = []\n",
    "    boost_label = []\n",
    "    optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
    "    train_best = float('inf')\n",
    "    best_model = None\n",
    "    \n",
    "    print('--'*8 + f'boost round: {boost_num}/{boost_epoch_num}' + '--'*8)\n",
    "    print(train_tensor.shape, y_train_tensor.shape)\n",
    "    for epoch in range(epochs):\n",
    "        print('='*20 + f' Epoch: {epoch} '+ '='*20)\n",
    "        model_train(train_loader=train_loader, optimizer=optimizer, criterion=criterion, labels=y_train_tensor, model=model, boost=True)\n",
    "        loss = predict(val_loader, model, criterion=criterion, labels=y_val_tensor)\n",
    "        if loss <= train_best:\n",
    "            train_best = loss\n",
    "            best_model = model\n",
    "    torch.save(best_model.state_dict(), f'./best_model{str(boost_num)}.point')\n",
    "    # 开始boosting\n",
    "    train_tensor = torch.cat(boost_feature, dim=0)\n",
    "    y_train_tensor = torch.cat(boost_label, dim=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ed5ae33",
   "metadata": {},
   "source": [
    "# 模型stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "74c89c15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model0 = DNN().to(device)\n",
    "model1 = DNN().to(device)\n",
    "model2 = DNN().to(device)\n",
    "model3 = DNN().to(device)\n",
    "model0.load_state_dict(torch.load('best_model0.point'))\n",
    "model1.load_state_dict(torch.load('best_model1.point'))\n",
    "model2.load_state_dict(torch.load('best_model2.point'))\n",
    "model3.load_state_dict(torch.load('best_model3.point'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "27c5441b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model0.eval()\n",
    "model1.eval()\n",
    "model2.eval()\n",
    "model3.eval()\n",
    "preds0 = model0(torch.FloatTensor(val_sp).to(device)).argmax(dim=1).cpu().numpy()\n",
    "preds1 = model1(torch.FloatTensor(val_sp).to(device)).argmax(dim=1).cpu().numpy()\n",
    "preds2 = model2(torch.FloatTensor(val_sp).to(device)).argmax(dim=1).cpu().numpy()\n",
    "preds3 = model3(torch.FloatTensor(val_sp).to(device)).argmax(dim=1).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "588b0d85",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans0 = model0(torch.FloatTensor(val_sp).to(device)).cpu()\n",
    "ans1 = model1(torch.FloatTensor(val_sp).to(device)).cpu()\n",
    "ans2 = model2(torch.FloatTensor(val_sp).to(device)).cpu()\n",
    "ans3 = model3(torch.FloatTensor(val_sp).to(device)).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "53a7b8a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3131649788488655"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(preds3 == val_label).sum()/len(val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "76d48896",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature = np.stack([preds0, preds1, preds2], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "844f0aa3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23403, 3)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "6ab1d554",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "a91f7f50",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(feature, val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "5686cc76",
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import Pool, CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "49f27314",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CatBoostClassifier(#loss_function=\"Logloss\",\n",
    "                           eval_metric=\"AUC\",\n",
    "                           task_type=\"GPU\",\n",
    "                           learning_rate=0.1,\n",
    "                           iterations=50,\n",
    "                           l2_leaf_reg=50,\n",
    "                           random_seed=43,\n",
    "                           od_type=\"Iter\",\n",
    "                           depth=5,\n",
    "                           early_stopping_rounds=1000,\n",
    "                           border_count=64,\n",
    "                           loss_function='MultiClass',\n",
    "                           #has_time= True\n",
    "                          )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5fd7eee2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "AUC is not implemented on GPU. Will use CPU for metric computation, this could significantly affect learning time\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\ttest: 0.8581913\tbest: 0.8581913 (0)\ttotal: 68ms\tremaining: 3.33s\n",
      "1:\ttest: 0.8734694\tbest: 0.8734694 (1)\ttotal: 156ms\tremaining: 3.73s\n",
      "2:\ttest: 0.8753030\tbest: 0.8753030 (2)\ttotal: 237ms\tremaining: 3.72s\n",
      "3:\ttest: 0.8783921\tbest: 0.8783921 (3)\ttotal: 344ms\tremaining: 3.95s\n",
      "4:\ttest: 0.8863586\tbest: 0.8863586 (4)\ttotal: 429ms\tremaining: 3.86s\n",
      "5:\ttest: 0.8881551\tbest: 0.8881551 (5)\ttotal: 506ms\tremaining: 3.71s\n",
      "6:\ttest: 0.8926710\tbest: 0.8926710 (6)\ttotal: 592ms\tremaining: 3.64s\n",
      "7:\ttest: 0.8927174\tbest: 0.8927174 (7)\ttotal: 686ms\tremaining: 3.6s\n",
      "8:\ttest: 0.8940856\tbest: 0.8940856 (8)\ttotal: 774ms\tremaining: 3.52s\n",
      "9:\ttest: 0.8958414\tbest: 0.8958414 (9)\ttotal: 879ms\tremaining: 3.52s\n",
      "10:\ttest: 0.8964050\tbest: 0.8964050 (10)\ttotal: 960ms\tremaining: 3.4s\n",
      "11:\ttest: 0.8967690\tbest: 0.8967690 (11)\ttotal: 1.03s\tremaining: 3.26s\n",
      "12:\ttest: 0.8978529\tbest: 0.8978529 (12)\ttotal: 1.1s\tremaining: 3.12s\n",
      "13:\ttest: 0.8980099\tbest: 0.8980099 (13)\ttotal: 1.16s\tremaining: 2.99s\n",
      "14:\ttest: 0.8989150\tbest: 0.8989150 (14)\ttotal: 1.23s\tremaining: 2.88s\n",
      "15:\ttest: 0.8993402\tbest: 0.8993402 (15)\ttotal: 1.3s\tremaining: 2.75s\n",
      "16:\ttest: 0.8996862\tbest: 0.8996862 (16)\ttotal: 1.36s\tremaining: 2.64s\n",
      "17:\ttest: 0.9002900\tbest: 0.9002900 (17)\ttotal: 1.43s\tremaining: 2.54s\n",
      "18:\ttest: 0.9007042\tbest: 0.9007042 (18)\ttotal: 1.49s\tremaining: 2.43s\n",
      "19:\ttest: 0.9010417\tbest: 0.9010417 (19)\ttotal: 1.55s\tremaining: 2.33s\n",
      "20:\ttest: 0.9017178\tbest: 0.9017178 (20)\ttotal: 1.61s\tremaining: 2.22s\n",
      "21:\ttest: 0.9027838\tbest: 0.9027838 (21)\ttotal: 1.68s\tremaining: 2.13s\n",
      "22:\ttest: 0.9033130\tbest: 0.9033130 (22)\ttotal: 1.73s\tremaining: 2.04s\n",
      "23:\ttest: 0.9035994\tbest: 0.9035994 (23)\ttotal: 1.79s\tremaining: 1.94s\n",
      "24:\ttest: 0.9038694\tbest: 0.9038694 (24)\ttotal: 1.86s\tremaining: 1.86s\n",
      "25:\ttest: 0.9039295\tbest: 0.9039295 (25)\ttotal: 1.93s\tremaining: 1.78s\n",
      "26:\ttest: 0.9043766\tbest: 0.9043766 (26)\ttotal: 2s\tremaining: 1.71s\n",
      "27:\ttest: 0.9045276\tbest: 0.9045276 (27)\ttotal: 2.06s\tremaining: 1.62s\n",
      "28:\ttest: 0.9047814\tbest: 0.9047814 (28)\ttotal: 2.13s\tremaining: 1.54s\n",
      "29:\ttest: 0.9050033\tbest: 0.9050033 (29)\ttotal: 2.2s\tremaining: 1.47s\n",
      "30:\ttest: 0.9054341\tbest: 0.9054341 (30)\ttotal: 2.27s\tremaining: 1.39s\n",
      "31:\ttest: 0.9056523\tbest: 0.9056523 (31)\ttotal: 2.34s\tremaining: 1.31s\n",
      "32:\ttest: 0.9058112\tbest: 0.9058112 (32)\ttotal: 2.39s\tremaining: 1.23s\n",
      "33:\ttest: 0.9059432\tbest: 0.9059432 (33)\ttotal: 2.46s\tremaining: 1.16s\n",
      "34:\ttest: 0.9060708\tbest: 0.9060708 (34)\ttotal: 2.52s\tremaining: 1.08s\n",
      "35:\ttest: 0.9062955\tbest: 0.9062955 (35)\ttotal: 2.6s\tremaining: 1.01s\n",
      "36:\ttest: 0.9064224\tbest: 0.9064224 (36)\ttotal: 2.66s\tremaining: 935ms\n",
      "37:\ttest: 0.9064522\tbest: 0.9064522 (37)\ttotal: 2.75s\tremaining: 867ms\n",
      "38:\ttest: 0.9066250\tbest: 0.9066250 (38)\ttotal: 2.81s\tremaining: 792ms\n",
      "39:\ttest: 0.9066875\tbest: 0.9066875 (39)\ttotal: 2.87s\tremaining: 717ms\n",
      "40:\ttest: 0.9067206\tbest: 0.9067206 (40)\ttotal: 2.93s\tremaining: 644ms\n",
      "41:\ttest: 0.9070910\tbest: 0.9070910 (41)\ttotal: 3.03s\tremaining: 577ms\n",
      "42:\ttest: 0.9072158\tbest: 0.9072158 (42)\ttotal: 3.11s\tremaining: 506ms\n",
      "43:\ttest: 0.9073845\tbest: 0.9073845 (43)\ttotal: 3.18s\tremaining: 434ms\n",
      "44:\ttest: 0.9074748\tbest: 0.9074748 (44)\ttotal: 3.25s\tremaining: 361ms\n",
      "45:\ttest: 0.9076261\tbest: 0.9076261 (45)\ttotal: 3.33s\tremaining: 289ms\n",
      "46:\ttest: 0.9076391\tbest: 0.9076391 (46)\ttotal: 3.4s\tremaining: 217ms\n",
      "47:\ttest: 0.9077120\tbest: 0.9077120 (47)\ttotal: 3.48s\tremaining: 145ms\n",
      "48:\ttest: 0.9078907\tbest: 0.9078907 (48)\ttotal: 3.59s\tremaining: 73.2ms\n",
      "49:\ttest: 0.9079729\tbest: 0.9079729 (49)\ttotal: 3.69s\tremaining: 0us\n",
      "bestTest = 0.9079729261\n",
      "bestIteration = 49\n",
      "0.6239958981370706\n"
     ]
    }
   ],
   "source": [
    "model.fit(\n",
    "    x_train, y_train,\n",
    "    eval_set=(x_test, y_test),\n",
    ")\n",
    "preds = model.predict(x_test)\n",
    "print((preds.flatten() == y_test).sum() / len(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae01014d",
   "metadata": {},
   "source": [
    "# boosting test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "681e525b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'best_model1.point',\n",
       " 'boost_dnn.py',\n",
       " 't.log',\n",
       " 'eda.ipynb',\n",
       " 'best_model.point']"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "34601891",
   "metadata": {},
   "outputs": [],
   "source": [
    "model1.eval()\n",
    "model2.eval()\n",
    "model1 = DNN().to(device)\n",
    "model2 = DNN().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "af2b6ea3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.load_state_dict(torch.load('best_model0.point'))\n",
    "model2.load_state_dict(torch.load('best_model1.point'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fd572cf1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The text.latex.preview rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The mathtext.fallback_to_cm rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: Support for setting the 'mathtext.fallback_to_cm' rcParam is deprecated since 3.3 and will be removed two minor releases later; use 'mathtext.fallback : 'cm' instead.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The validate_bool_maybe_none function was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The savefig.jpeg_quality rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The keymap.all_axes rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The animation.avconv_path rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "06/14/2021 12:50:14 - WARNING - matplotlib.style.core -   In /home/zwl/miniconda3/envs/asr/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The animation.avconv_args rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3f24054d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_cm(preds, label, file_name, n_classes=10):\n",
    "    '''\n",
    "    label:真实标签，一维ndarray或者数组都行\n",
    "    preds:模型的预测值\n",
    "    n_classes:看问题是几分类问题, 默认是10分类问题\n",
    "    '''\n",
    "    cm = confusion_matrix(label, preds)\n",
    "    def plot_Matrix(cm, classes, title=None,  cmap=plt.cm.Blues):\n",
    "        plt.rc('font',family='Times New Roman',size='8')   # 设置字体样式、大小\n",
    "        \n",
    "        # 按行进行归一化\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        # 占比1%以下的单元格，设为0，防止在最后的颜色中体现出来\n",
    "        for i in range(cm.shape[0]):\n",
    "            for j in range(cm.shape[1]):\n",
    "                if int(cm[i, j]*100 + 0.5) == 0:\n",
    "                    cm[i, j]=0\n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "        im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "        # ax.figure.colorbar(im, ax=ax) # 侧边的颜色条带\n",
    "        \n",
    "        ax.set(xticks=np.arange(cm.shape[1]),\n",
    "            yticks=np.arange(cm.shape[0]),\n",
    "            xticklabels=classes, yticklabels=classes,\n",
    "            title=title,\n",
    "            ylabel='Actual',\n",
    "            xlabel='Predicted')\n",
    "\n",
    "        # 通过绘制格网，模拟每个单元格的边框\n",
    "        ax.set_xticks(np.arange(cm.shape[1]+1)-.5, minor=True)\n",
    "        ax.set_yticks(np.arange(cm.shape[0]+1)-.5, minor=True)\n",
    "        ax.grid(which=\"minor\", color=\"gray\", linestyle='-', linewidth=0.2)\n",
    "        ax.tick_params(which=\"minor\", bottom=False, left=False)\n",
    "\n",
    "        # 将x轴上的lables旋转45度\n",
    "        plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "                rotation_mode=\"anchor\")\n",
    "\n",
    "        # 标注百分比信息\n",
    "        fmt = 'd'\n",
    "        thresh = cm.max() / 2.\n",
    "        for i in range(cm.shape[0]):\n",
    "            for j in range(cm.shape[1]):\n",
    "                if int(cm[i, j]*100 + 0.5) > 0:\n",
    "                    ax.text(j, i, format(int(cm[i, j]*100 + 0.5) , fmt) + '%',\n",
    "                            ha=\"center\", va=\"center\",\n",
    "                            color=\"white\"  if cm[i, j] > thresh else \"black\")\n",
    "        fig.tight_layout()\n",
    "        plt.savefig(file_name, dpi=300)\n",
    "        plt.show()\n",
    "\n",
    "    plot_Matrix(cm, range(n_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "28d05f13",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06/14/2021 12:50:14 - WARNING - matplotlib.font_manager -   findfont: Font family ['Times New Roman'] not found. Falling back to DejaVu Sans.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAAEdCAYAAADXQ3g/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABgK0lEQVR4nO2dd3gUVfeA39mULQkt9N5bgoAgRTokISR0SCgiSJGiYkFFaQJSbPQqItjQj96LlNARBSlSlU9FQEGQmi3ZJFvm98dsGpKQTXIp3+++z+Pjhp09c/bs7Nkp986rqKqKRCKRiED3sBOQSCT/u8gGI5FIhCEbjEQiEYZsMBKJRBiywUgkEmHIBiORSITh+7AT8A/IpxqDigmJreCmdAETOp2YPup2uwGExQdISHICoAhah6+iDVPQ+fgIie92ubT4OkHx3S5cqio0vp9OJ7Q+TrcqdBv10SlC458+dfKGqqqF7/X8Q28wxqBi1HnzcyGx/d0JfPFsXQLz5BES32qxoNMpwuID/PTb3wAYTIFC4hc1aU1S1HuIizMDEBAoJr7NasGS4MAUICZ+vM1C0XxG8giqj8Vi4aYlEZOg+sRbLeQ3+Qn9DhQvnP9iRs/LQySJRCIM2WAkEokwZIORSCTCkA1GIpEIQzYYiUQiDNlgJBKJMB65BlM0j57VA59iRtcQPuoUjI9OYW63J9jyQgNK5DMAoACT2lVjTkwNiubRA/Bqi/IE+Hs3VuHHw4cIa9GEiNDmjBj+OmazmfaR4bRrE4bZrF1eHTKwHy7PWA5v+fvKFRo3qEvBvEacTidms5m2EWFERYSmxB/8vHfxr1/7m17tmtKoahGcTifnzp5kUI+2DOrRlg5Nn+A/n83H7Xbz+sCe9I9uzd9/XQLgw3fewGqO8yr/s2dO07plEyLDmvPSoAHExcXRMSqcDpGp9XlxUP9s1+fSxQsEVyhJx8hQYjpGYjGb6dKuNZ3bhmPxxB862Pv4165eoUPo01QvXQCnUxtHtHDuDLq1C2XYkH44HA7cbjeD+8QQ07YVl//UajTu7deweFkjkTidToY+35vuHVvz3vhRWMxmenZqQ4+OESn1eeOl57Nd/x8PHyK0RRNat0rd/tu1CadtRFi2t8+7UUTdD0ZRlBnAU8AxVVVfzWi5fKWrqmnHwRTNo2dAozK8t+3XlH8rYPJjUOOyLDn8F1fiEqhSJIB6ZfPz019mgovl4cdLd2hYLj/Ljl5JF/t+42CuXb1Kvvz5MRgMDOj7LJUqV6FixUoA6PV69AYDFrOZmO497/n6+42DSUhIwG6307NbFzZ9u4NNG9eTlJio5abXY9AbMFvMdMsgPvx7HExiYgKJCXbeHPws879ej69v6lCm1wf2ZNjoSVgtZr7ft5O6DZty6thhGjYL5bvd23luyGv/ip/ZOBiHw4Gfnx8ALw0aQOWqVSlZqnRqffR6LBYL0d16ZJh/ZuNgLl28wPsTx/Lxoq8A2Lh+Tbr66PUGrBYzXWIyjn+vcTCJCQkkJNh5oW8Pvlq1mbg7txn+8kA+W7qOT2ZPo3S58pQuW479u2Np0KgZx48comnLMPbEbmPwy6+ni/8wx8FsXr+aC+d/56VhbzH27WEEFSxIuQra9unvr22fVouZjl27Zxg/s3Ew6bb/5zzbf6Wsb/+QMg7mqKqqT93reSF7MIqi1AECVFVtCvgrilLPm9fXLpWXWdE1iH6yOAC34x3pnk9wuPH30WHw05HgdNG5VjHWnbjqdZ5FixXDYND2inx9fSlYsBCJiQnEx9swmkysW7OKrjEZf3j3w2AwUKBAgZS/TUYTCQkJ2Gw2TEYTa9esItrL+Hq9gbz5Cvzr3+3xNm5ev0bpchUxGE0kJSZij7dhMJlY8dVCuvUZ6HX+yc0FtC98qdJlSExIIN5mw2g0sn7tarpEd/M6bloO7NtLu9YtWDB3JkZPfbT4JjasXUWnrt7H1xsM5MufWqOTx4/QoFEzABo1b8lPRw5jNJpITEjAHm/DZDKxZPECevcfnKP3kttcuvAH1UKeACD4iZoULFSExEQtZ6PJxJb1q2nfOSbb8dNu/z6+vhQsVChl+zSaTKxdnbPtH8QdIj0NxHoexwINs/rCW/FJ9PnyOMNWn6Zu6fxUKGT61zKXbtvx0SmEVi3MX7ftXLplJ/rJErzcvDwmLw+TAE6fOsnNmzfoN2Agx44d5eTJE1jMZiLaRDFuzEjeGfV2yq52TmgZGsaxo0c4eeInzBYzEZFRjB09gjEj38px/O/27KBR8zAAyleqisvlZNuGVZQtX4lyFavwzeL5TBn/FlaL2au4WzZt5OmnanHj+j907NyV48eOcurkCSwWC63bRDH+nZGMHZ29+hQtVpwfjp9l3ZZY9u7eRbHixTlx/CinTp3AYjET3iaKiWNH8e6YETmqjzkuLmUPJE+efMTF3aZSlWq4XE42rFlOuYqVqVi5Kp99Mod3R72BxcsaiaJCpSocOrgfgO/37+XWrRucPH6MM6dOYrWYadU6kg8mjOG98aNyVJ+023/K52s20yYyirFjRjImB9u/qAaTH0j+lOKAdD+5iqIMUhTliKIoR5Jsd9K90OFSSXC6cavw/R+3KF/w3w0GYNHBS3y04zdCqxbmp7/i0Cmw99ebhFYp5FWit27d4s1hrzDv40X4+fkxY9Y8pk6fzY7tWwkIDKR6SAjVQ0LYs3unV3HvhZ+fHzPnzGfazDns2LaVwIBAgkNqEBxSgz27chZ/z/ZNtIxon/L3S8PHMW7KfLauX0ndhk1QVTehkR3ZtnGVV3Gj2rXn+yMnKF6iJNu2bGbarLl8NH0Wsdu3EhAQQHBwDYKDa7A3G/XR6/UEBATg6+tL68gozv18lo9mzOWDqbPYuX0rAQGBVAsOoVpwCPv27PI6fjJ58+XDYrEAYLWayZs3PwBvjp7AR7MXsnHNcho0aorqdhPZvjMb16zI9rpyk7A2bUmw2+nZqQ3+ej3Fipdg8tTZTPhwBntit2EKCKRqtWCqVgvmu327s7WOW7du8cZrrzB/gWf7nz2PqTNms32btv0HB4cQHJz97V9Ug7kD5PU8zuv5OwVVVReqqvqUqqpP+QfkT/dCo19qSjVK5OXKnYQMVxJSPA/n/rHio1Nwq+BGxeCX9T0Yp9PJwP69mfjehxQtljrhcuXypXSJ7oY9Ph6dTodOp8NmtWY57v1YuXwpXaO7EW9PjW+1ZT++0+Hgj9/OUSX4iXT/fuLoIarXfBK3y4WiaOux22xZjpvoOR8CkCdPHgxGbXd61YpldO4ag91uR9HpUHQ6bF7ETcbq+dIDHP7hIOXKVwBgzcpldOraDXua+thyUJ8natfl8PfansB3e3dT+6n6Kc8dPfwDT9Sqg8vtQtHp0Ck64rPxXkTg4+PDhA9nsHTdVnx8fGjWQttDXb96Oe06x2CPj0+tfza2T6fTycB+vZn0/r+3/64xubP9i5rs+D0wGFgBhAFfZPWFNUvmpX/DMiS53Jy+YuHna1bGRVahRom8lMpvYNnRy3x3/jYAbUOKMG3XeVRV5bkGpWlQLj+Ttv56nzWksnb1So4dPcK4MSMBGDdhMk/Vq8/uXbEs+PRz4uLieKZbF1RVZdmqdVmOm4zD4aBz+yhOnzxBx7ZtGD9xMnXqPsWunbF8skiL3yOmM6qqsmL1+izFdDocvNKvK7/+fIqXn+vMS2+Ow2K+Q72nm/1r2XXLv2L05JkoOh0LZ37AwT3bmTRrUZbz37l9G/PmzACgYsXKtAprjcvlYs+uWOYv/Iy4uDie7a7V5z8r12U5bjLfHzzAB5PGoffX0+DpxtSt1yAl/txPPsMcF0efnl1RVZWvl6/NclyHw0H/Hp34+cwp+nbvwJujxlO/YRO6tQulRKnS9Bs8NGXZFd98zqSpc9HpdMyeMpk9sduYueALr9+LCK5eucyrQ/qiKDq6du9F8ZKlcLlcHNizk2nzFmE2xzGwdwyoKou+Xu11/LWrV3L06BHGjta2//ETte0/7fbZM0b7fJevXpet9yDyKtIsoA5wQlXVoRktd/dVpNxEzqa+P3I2debI2dSZc7+rSMJu15DZpWmJRPL/g0duoJ1EIvnfQTYYiUQiDNlgJBKJMGSDkUgkwpANRiKRCEM2GIlEIoyHbhVQcOPvzni0bk7wcyfmaATo/bDZrOgURVh8gES72FGlNtUtNH68wPonx7cn5nyeWEbY423YfLJ/u4L7YbNasccnCYtvj7fhp4r7mt/v+/XQG0zpAia+eLaukNg2m5Wwvh9hSxQzmDBAr7D7q7cR2WPKeiZ7BgQECIkfl2a4vgiSnFoD83WKaWRJTjdBAf4EBOiFxLcpjvsvlEPyGn0JMPndf8Fs4Kf6oqogaDztfeM+9Aaj0+mEjoS1JapYE0T9SusICAwUNsoTwO3WPkFRNXImxxc0ktSepP36ixrJCxBg8BW6Dfn46IR+xi6XW2j+qipu+7kf8hyMRCIRhmwwEolEGLLBSCQSYcgGI5FIhCEbjEQiEYZsMBKJRBiPdIMR4S0yGvxYM3sI2z59lRUzBuHv58uwPqHs/GwYn09+Dl9fHYqisGLGIHZ9PowyxbXbCc8Y0Y28gQav8r9y5QpP169LgTypXqSoiDAiW6d6kQYNyJl3qUnDpyiUz5QSPze9NmfPnKZ1qyZEhjfnpcG570XaHbuN6HbhRLcL58lqZVm57Gu6dYwgpkOq9+e1Fx9t749IROcv2gsGgu5opyhKCWATEAwEqqqa4VDLWrWfVA/8cPSez+WGt6hmh3fSjYPp2KoWwZWK8/7Crbw1IILL/9whunUdOr/8MW/0DeOPv27yx+UbhD1dnf1Hf6VBzfLsOPgzbZqEMP3L2HTxAw06zmyelOEYiWQvUo+YLmzeuoNNG9an3Oc2bf7demTsnUm+I9y9xjEkx3+mW1c2frudzRvvHT8zr82d5Pj3GKeSzos0eACVq9zlRTLosZgz9yJdv3UHuP84mHZhTegzYDB+vtr6/D3eJavFQqfojNUZNquFPBmMg8kt74/IcTAWiyXDcTC5lX9G42By+v1Kjl+iyAP2IgG3gFDgh5wEEeEtOv/XDfSeL03+PEbKFg9i/xHtPr67Dp2jfs1yxCckYdD7YTLosdmTGNKjOQuW7/M6/7u9SEaTiYTEBGzxqd6Z6G65510ymky56rXJ0IsU7/Eircm5Fwng4oXzFCpchEKFCqd+vkYjm9avoUOXR9v7IxLR+Yv2goGgBqOqaoKqqrdzK15ueot+u/QP9Z4oy9FVo6kTXIbzf17HbNPmQsVZ7eTPY+LcH9fw9dHRPfIpfr34D//94yovP9uSaW9FkyfAu8OktLRK40VK9s68M2oEo0fk3IsE0LJVWK57bbZsvsuLdDz3vEjJfLtxHZHtOtK0RSgnjh/lzCktfmjrSN4bP5pJY0c+st6fB4Ho/EV6wYTd9BtAUZQ9QNjdh0iKogwCBgGULFW67s+/Xsgwxq1bt3imWxe+/HpZilrB7Xbz4uABdInuxo0b1wEoUqQoYeER6V57r0OkgTFNCDTqmfHVTl7rHcr1O1aKBuVh+pex1K5Wip5t6/P2tDUpy88f+wwLlu2lbYsn2HfkV6qVL8bi1d8B9z9ESiYirCWbt+5I0by63W6GDBxA15hu3Ljuyb9oUcJbR/zrtZkdIiUTGd6Kjd9uTxf/hUH/jn93fSDzQ6S0vPXGqzRr3pJ2HTrhdrsZOuR5OneN4eaNGwAULlKE0HvEz8ohUte2YXy6ZDlBQQVT8n996CA6dI7m5k1P/MJFaBHa+l+vzewQCbTtp2dMF776Jv32k9X6PMxDpNzKP7OpAjn5fiXHfxiHSJmS1otUsGDBDJcT4S1SULhljgfgxh0rZYsH0aSudtzZqkE1Dp/8I2XZhrXKc+zsJXx8dLjdKqpbJdCU80l1K5blnnfmXuSW1+ZfXiRD7nqRAP65dhU/f7+U5gKwbtVyOnSOxm63o1M0V1F24j8I749IROf/ILxgD32yY2aI8BYt//YISz7szzNt6+Nwuuj99mf079KInZ8N48+rt5nzTaohr2+nRgydvBS3W2X04CjaNAnhuZFfZDl/h8NBp/ZRnDp5gg4eL1Jdjxdp4WIt/+7Rmhdp5ZqseZHujt+lQ1tOnzpBp3aRjJsw6V/epZx4bXbuyIIXqYfHi7TC+/gA27ZsJCIy1UjpcrnYt2cnM+cvwhwXR/9nNe/P5/95NL0/IhGdv2gvGIi7iuQHfAvUBY4Bo1RVPXSvZTO7ipRT7nWIlJtk9RApJ2TlECknZPUQKbtk9SpSdrnfIVJOediHSDnlfodIuRE/s0MkIXswqqo60IyOEonk/zGP9EA7iUTyeCMbjEQiEYZsMBKJRBiywUgkEmHIBiORSIQhG4xEIhHGQx9o53a7sQpSZ9hsVgL0CqL6aIBeET4CVKTX6UHEfxBeJJ1LrPfHRyfud9hmteJyi3NT2WxWYcqS5PiZ8dAbjMutckuQeMpud7D7y7cJCAwUEt9mtXLubwt6o7gNRHXYAdDbxMiXChk96xESHfIYtBnZAUYx3h8flx/T9p7H7Zv9SaiZoXMmMLxlRSGxk3G5VVxuMZ+Ay62iAi5BXeZ+cR96g1F0OkwBAp05gXqhThi90Y3BJKaBAbiTtF9PUeswmbTmKGokb5KPlr/Iz8Dta8DtZxQWX7T7ys/hEutFQtznez/kORiJRCIM2WAkEokwZIORSCTCkA1GIpEIQzYYiUQiDNlgJBKJMB65BnPt6hXat3qaaqXyp9xoeOHc6cS0bcVrQ/ricDhwu90M6h1DTFRLLv95EYCxb72G2Rzn1br+vnKFRg3qEpT3Lm9RRBpvkZfemTM/HWFQt9a80DOSWZNHAfDNp7MZ0qMN418fiNOT/1tDnmFw9wj+vnwJgKnj38RquX/+16/9Ta92TWlUtQhOp5NzZ08yqEdbBvVoS4emT/Cfz+Zr97Qd2JP+0a35+y8t/ofvvIHVy/okM2/2DCLDmmE2m+kUFU7HNF6kl3LgRfr7yhUaN6hLwTT1bxsRRlSa+mfX+xNWuSDDW5ZDp8Dbrcozq3M1Cgf4A6AALzQqzfCW5QkyaeNzej5ZHIPvo/V1OHvmNK1bNiEyrDkvDtK8VB2iwmmfpv4v5KD+or1XIGgcjKIoDYAZgAs4oqrqsKy+Nn/+IL5es4Uhz2m6hJs3rvP9gX2s3LyLBbOnsmPLBkqXK0+tOvVo2Lgp325cS9OW4ZQsXZq8efN5lWeBoCA2b42lZ7cuAOzauYPn+vVPeWzQGwgLb42Pj0+WYxYrWZo5Szag1xsY//pAfvrxIMcO7WfBsq0s+WQm+2I3U6J0OUJq1aVOgybs3rqBBk1bUaxEaQLz3D//vPkL8PE3G3hz8LMAVA2uycJlmwF4fWBPmraK4NyZE9SoXZe6DZuy89v1NGwWSrGSpQn0sj6g3Zf3zKmTAOzZFcuzz/VPeWww6GkV5l190lIgKIhN96l/qJf1B/DVKZTKrw28c6sw/7tLdKlZNOX50vkNXLhl59x1G3VK5eXsVSs345NIcIobMJkdKlepyvbdBwB4cdAAPl/0Cb099d+9KxaDXk9oDupfuUpVtu/S4r80WIuf9vPV5/DzBXF7MBeBVqqqNgWKKIryRFZfqDcYyJc/1fVz4tgRGjZuCkDj5q04fvQwRqOJpMQE4uPjMZoC+GrxAnr3H+J1kv/yChk1r1C8zYbRaGLNmlVEe+mFKVi4KHp9qsvm93NneLJ+EwDqNW7B6Z+OYDAYSUpMxB4fj9FoYvWST+n67PNZiq/XG8ibr8C//t0eb+Pm9WuULlcRg9HkiW/DYDKx4quFdOsz0Kv3kcySLxbTo1dvAIxGo+blSeNF6pwDL9Ld9TcZU71OJqOJtdmoP0CT8gX4/sKdlL8tiel/gZNcbnx9FPQ+OpKcblpUCmLPb7ey/T5EkdZLpfd4qZK3T5PRyLq1OfNSPQjvlSgv0lVVVRM8fzrR9mSyhdl8h8A8eQHIkycfcXfuUKlKNZxOJxtWL6d8xcpUrFSFzxbM4d2Rr2OxmLOdd6vQMI4ne4ssHm/R6BGMHum9t+i3X05z59ZNAvPkS7kfbWCevFji7lCuUlVcLifbN66idPlKlK1QhWWff8z0CW9jy2b+3+3ZQaPm2l1Ky3vib9uwirLlK1GuYhW+WTyfKePfwupFfIfDwXcH9tGsRSsAWrQK46fjRznt8SKFt4ni3XdGMi6HXqRkWqbxRpktZiIioxg7egRjvKi/ToHKhU2cu56xheCqJQkfRaF+mXxcsyRxzZJIWJWCdK9d7JE7TNqySfNSXfd4qX46luqlisgFL5Vo75XQaiqKUhMopKrq2bv+fZCiKEcURTly+9bNTGPkzZs/5UthtZjJm0/bzR8+ZgJT5ixkw+rlNGzcDLfqJrJ9FzauWZHtfP38/Jg5Zz7TZs5h+7atBAQEEhxSg+CQGuzetTPLccx3bjPt3bcY9f4cAvPmw2bVJnParJaUw5Qhb4xlzIfz2LFxFU82aILqdtOyTQd2bPL+7vkAe7ZvomVE6t35Xxo+jnFT5rN1/UrqNmyCqroJjezIto2rshxz+X++JrpbqjbUz8+PqTPn8uG0WezcvpWAgACqB9egenAN9u7Oen0yIm39d2zbSmCa+u/JYv0bls3P4Uv3P9e07vQ/fHnkCvXL5OPcP/EoKBz7y0y9Mt4fRookql17vj9yguIlSrJ1y2amzZrLR9NnsSOX6h/VNjX+tm83M23mXD6aNotYT/zg4BoE5yC+sAajKEoQMBcYcPdzab1IBYIy9iIB1HyyLocOaseJ3+3bxZN166c8d/Tw9zxRuw4utwudokPRKbkye3fF8qV0je6G3Z7GC5PFuE6nk3ffHMTQtydQsHBRqtesw/HDmqjtx+/2UKN26s3XTx07RLUatXG7XSie9cTHZ8P/43Dwx2/nqBKc/kj0xNFDVK/5JG6XC0XR4tu98Av99us5Pvt0AdEdo/jl57Ms/HguAKtXLKOTx4uUWp/seZHuxUpP/ePT1N+axfoXy6OnecUgXmlahuJ5DbSsFJThshUKGrl4245OARUVtwr6R2gPJq2XKm+ePBiNqV6qLl1jiE9Tf2s26v8gvFeiTvL6Al8Dw1VVverNax0OB/16dOTnM6d4rlt7ho9+l/pPNyambStKlCpNv8Evpyy7/OsvmDxtLjqdjlkfTWbPzq3MXPClV+tK6y16d+LkFK/QQo93pkeM5i1asTpr3qJd367j55PHmT9lPABD3hxL7XqNGNKjDUWLl6J73xdSlt2wYglvT5yBotOxePYHfL93O+9OX5RpfKfDwSv9uvLrz6d4+bnOvPTmOCzmO9R7utm/ll23/CtGT56JotOxcOYHHNyznUmzMo+flvGTPkh5HBnWjEEvDE3xIs1b+BnmNF6kb7LhRXI4HHRuH8Xpkyfo6PFG3e118rb+a05dS3k8vGU5dv92i4ENS1GpkIkigXq2n7vBiSvaHmWT8gX4+ugVVBXahxShRrE8LD70l9fvQxSx2+/tpdq9K5aPPV6qXt21+i9duc7r+A/CeyXKi9QTmA2c8fzTSFVVv7/XsiE1a6ubdv2Q6zkAxNssFBY4m9pqsXDmslnwbGrNQilqHUVEz6a2a798Ij+D8dv+K2w2tc5hZ0JkVaFepCSBs6mtFovQ2dRWq4XSRQs8cC/SUmCpiNgSieTx4dE54JRIJP9zyAYjkUiEIRuMRCIRhmwwEolEGLLBSCQSYcgGI5FIhPHQrQKq2028TYwXyR5vw4ZDSGzQtCWJ9twbwXovkrUloohXxc4gdtjjhca3Wa3onAn3XzCb6JwJQt1XNquVJGf2b4dw3/g2qzAlTXL8zHjoDcZHpxBk8hcS26Ym4VIFOmdUlWol8grzLgHU6zEPAJsYdRQ/fDEUAD+dGO9S8hQ5MdG1uH3qlsRoChAS356NqRve4nKrOAV6kYz+vuj9sn/Lhcxw+mYe96E3GJ1OJ9QJA2KdPL6+OqHOnOTGYk3MfLnsEhCgNUdRNdIpitD4AEa3r2C3llHoZ6wkOoV6i0x6X6H5Z4Y8ByORSIQhG4xEIhGGbDASiUQYssFIJBJhyAYjkUiEIRuMRCIRxiPdYH48fIiwFk2ICG3OiOGvYzabaR8ZTrs2qd6WIQOz5815EPFFEN6gCtvmDmTb3IGc3zCSod0ap/z986rhDO3WGEVRWPFhb3YtGEyZYvkBmPFGB/IG6L1a14+HDxHaogmtW6XWp12bcNpGhOXYWwRivFSgubU6hj1NSJkCOJ1OHA4HMVEtqFW+MBf/+B0At9vNkD4xdG/Xist/au6o8SNew5JNd5RI5s+ZQdvw5ljMZjq3a02ntuFYPPUZOjhn3iLRiLqjXQ1gIZpN4Degv5rBimrVflI98MPRe8a5dvUq+fLnx2AwMKDvs1SqXIWKFSsBmsZBbzBgMZuJ6d7znq+3WrQRwhmNwciN+KLHwZQOH6mt6x7jYPZ9+iKRryzCZtcGy6z4sDcjZm8hX6CesAZV2H/sPA2eKMOOQ7/S5umqTP9m379i/LFhBHDvGqWrz3Oe+lTKen0A4j2jYO8VPyEhAbvdTs9uXdj07Q42bVyfcp9YvV6PQW/AbDHTLZP4VouFq+aEdONgEhMSSEiw82K/Hny5cjM+Pj7cvP4PH016h5eGvU3Z8hU5feIY+3fH0qBxM44fOUTTFmHs2bmNQUNfT5+/zULx/OLGwVgsFuIzGQeTmJjI6y8P4cIf5xky9NV09dHrDVgsZrrG9MgwvtVqEToOxmKxUCQob4Z3tBO1B3NOVdVGHi8SwD1Xfj+KFiuWciNiX19fChYsRGKix9tiMrFuzSq6ZsOb86Dii6RciQJcu2VJaS4mgx/FgvJw/vJN4hMcGPx9MRn9sdmTGNL1aRasvucdSzMlbX18fH0pWKhQirfIaDKxdnXO6iPCSwX/dmspikKhIkXTLWMwmtJ91ks+W8Cz/QZn+72I4usvF9O9Vx9Aq09imvpsWLuKzl1z5i0SjSgvUtoJQInAnzmJd/rUSW7evEG/AQM5duwoJ0+ewGI2E9EminFjRvLOqJx5eUTHF0Gn5jXYsC/VBhPxdFW2//BfAM5dvI6vj47urWvx6583+O/F67zcownThrUnj8m7wyRIX5/jyV4es+aNGjtmJGNyqT656aW6H5WqVMPldLJxzXLKV6hMhUpV+fyTOUwY9UaO3Fq5icPh4OCB/TRr3hKA5i1DNS/VqRNYLGbC20QxYewoxo8Z8chtn8mI1JZ0UBTlNFAEuHnXcylepJs3M/ci3bp1izeHvcK8jxfh5+fHjFnzmDp9tuaFCQykekgI1UNC2JNNb4vo+KKIalKNzft/Tvm7Q7MQ1u89nfL3uE+2M3jyarqH12bf8fPoFIU1u0/RLbyWV+u5desWb7z2CvMXeOozex5TZ8zWvFGBgQQHhxAcnDv1yS0vVVZ5Y/QEPpy1kI1rltOgcVPcqps27TuzaW323Vq5yYqlX6c7/PHz82PKjLl8MDXZSxVI9eAQqgeHsG/ProeYacYIazCqqm5QVbUGcBlod9dzKV6kggUz9iI5nU4G9u/NxPc+pGixYin/vnL5UrpEd8Men8ZblI0Zr6Lji6JoUCBJDhe3zNpMZV8fHVXLFebUb+kNMQ1rlOHYL3/ho1NwqyqqWyXQi4mlTqeTgf16M+n9f9ena4y4+uTES+Utx378gRq16+B2aW4tnU5HfC46nnLCb7/+l88Xf0K3zm355ZezfLrA46VauYxOXbX6pHqLHp3tMy2ivEh6VVWTT0uagWzdc2Dt6pUcO3qEcWO0E53jJkzmqXr12b0rlgWfat6cZ7pp3pZlq9Y9cvFF0a5pMJvS7L20eKoie4/+/q/l+naox9AP1+JWVUYPCKPN01V5btyyLK9n7eqVHD16hLGjtfqMn6jVJ623qGeMVp/lq9d5/T5EeKmS4z7fsxO/nDlF/+4deH3UeD5bMJujhw5y8fxvDHxpGGGRmgVzxdefM3Gq5taaPWUye3duY/rHX3j9XkQwbuL7KY/bhjdn4BDNS7V3VyxzP9G8VL17dtW8VMvXPsRMM0bUVaSOQPLp+F+BQap67xuPZHYVKafc7ypSbsR/mFeRcoPMriLlBpldRcoN7nUVKTd52FeRcsrDvookyou0Hsj6T45EIvmf5JEeaCeRSB5vZIORSCTCkA1GIpEIQzYYiUQiDNlgJBKJMGSDkUgkwnjoVgG3250yXiW3ET260Waz4usjtkcHiDG6pCC6RnbBo2JtViv2eEGDhPC4tXwFeousVuxJ4uYRxdusqA5xX/P7jeB+6A0GECaGUgGny02SS4xczOFys/nnq+j0RiHxATbMGQKAKUCMe+nnS9cB8DeK+RIVMWqfrsNHjAAvPsGBv07BX5DXySUoblqsiU5cgupjT3Ticqu4BdY/Mx56gxHtRXK43AQIdM7o9BZ8DCZh8ZMbi6j34G/U9jD0RjENzBSgNRiRn4Hb5cYkMH5AoEHoaG2ry0eo18nk7yu0/pkhz8FIJBJhyAYjkUiEIRuMRCIRhmwwEolEGLLBSCQSYcgGI5FIhPFIN5gjhw8R3qIJbUKbMzIDb9ELOfAWXbp4geAKJekYGUpMx0gsZjNd2rWmcw68M1fOn2Pq4K5MfyGGJZOHc+PKn0x/IYbpL3bj8/Gv4na5cLvdLHh7INOGRHPz6l8ALJ/2Dnardzeb3h27ja5tw+naNpzaVcuycunXdOsQQUz7iJT8X3vhea/yP3viCC/1aMPLvdoy9/3R/P3XRV7u1ZZXnm3HxDcG4fLkP/rFXgztGcnVy9r93GdOGI7Vy5tl747dRnS7cKLbhVOnWllWLfua7h0j6NYhNf9hL3qXf1qcTidDn+9N944RvDd+FBaLmZ6dI+nRqU3Kjb3fGDrwkfMKXbt6hQ6hT1O9dKrXKTqyBU+UK8yF86lep8F9Yohpm+p1Gve2916nFUuXEN0+gs5tw/jz0kWi20fQtV3rlPq/8sKAHNVHyB3tUoIryutAF1VVm2S0TK3aT6r7s+BFev4ub5G/Xo/B4+WJzsRblNk4mEsXL/D+xLF8vOgrADauX0OSxzvj7/HOWC1mumTgnbFZLWw4+3e6cTAupwMfXz8AlkweTrMuz1K4VHlMefKy4ZMplA95knyFinL20D6q1GnI+dPHCG7QjDPf7ya815B/raN1WS33+41jaBvahOcGDMbXT1u33l+P3qDHYrHQOTpj9cepP7T7+CaPg7l5/RqBefOh1xuY9OZgOvToS/kqweTJm49FMyYRXOspChYpxo8HdlG7fhPO/PQj9Zq04tC+HfR8/pV/xS+exXEw7cI8+Xtq56/XY9Br+XfKJH+b1ZLhOJjN61dz4Y/zvPTacMaOGEZQUCHKVaiYEj/58+3YNeP48VYLBfOKGwdjsVi4Fme/p9fphb49+GpVqtfpw4ma16lchYqcSvY6NfJ4nVqGsSd2G4Nf/rfXKaNxMH9fucxHk99lxryFAGxav4akJM/27695r6wWM52jM/Yu2awWKpYs+MC9SCiKoge8u4X9XdzLW5SQmIAt3obJ4y3qkkNv0YF9e2nXugUL5s78l5dnw9pVdPLSO5PcXAB8/fzJX6QEpjx5AdD5+KLofPA3GHEmJZJoj0dvMLF31Vc069In2+/h4oXzFCpchEKFC5OYkIA93obRZGTTujV07BLjVayChYui13tcSD4+5M1fgDx583n+9kXn44PBaCQpKZEEuw2DycTabxbR6ZkBOcq/cOEiFCxUONVVZDSyaf0aOniZf1ouXfyDasE1AAiuUcsTPxF7fDxGo4ktG9bQvnP244siK16nZEeS3fNdWLJ4Ab37e+d12r1zBy63i+j2EYwa/hp6g4GEhETibfEYTSY2rltNxy458y6JPER6HvgyNwIle3n6pvHymM1mWufQW1S0WHF+OH6WdVti2bt7F8WKF+fE8aOcSuOdmTh2FO966Z05uX8Hk56NwHL7JoH58gNw5/o1zh35jur1m1KsXCVcLhdHdmygSJnyFC1bkV3LF7NixnjsNu/nZW3ZuI7Idh1p2iKUkz8le3MshEZEMnn8aCaOHel1fX4/d4a4O7coV6kaADeu/c3R7/fyVOOWlK1YFZfTSeym1ZQuV5EyFSqz8suPmT1pBDYvD/MAvt24jjbJ+R8/ytlTJ7BaLIS2juS98aOZlI38ASpUqsKhg/sB+P7AXm7dusnJn45x5vRJrBYzrcLb8MGEMbw3ftQj6xXKiEpVquFyOdmwZjnlKlamYuWqfPbJHN71wut0459rOJKSWLVxG0ajEZvVyolk75LZTFjrSCaNG8WEd7LvXRLSYBRF8QOaq6p6T1mLt16k4cNeYa7HWzR91jymTJ9N7PatBAYGEhwSQnA2vUV6vZ6AgAB8fX1pHRnFuZ/P8tFd3plqwSFU89I7U7NpOGO+3kb+wkU59d0uHEmJLJn0Bs+8/T4+vtrsjI5DhtN79BSObF9P5ScborpVnmwZyZEdG71+Hzu+3ULrqHb4+fnx/rQ5TP5oJrt3bCMgIIBq1UOoVj2EA3uznr/5zm1mTXyb4ZNmAZCUlMj7I1/izYkz8PXkP/D1dxjx/lxiN62mdv3GqG6V5hEd2Llpjff5b91C60gt//emzWHiRzPZHbsNU0BASv29yT+ZsIi2JCTY6dk5En+9nmLFSzB5yiwmfDCdPTu3YwoIpGr1EKpWD+G7fbu9jv+weXP0BD6a7fE6NWqK6nYT2b4zG9dkzeuUJ28+nm7cDIAmzVpy4fzvfDh9Du9Nmcmu2G0EBKZu//uz6V0StQfTG/hPRk9640UadA9v0SqPtyg+h16etLO4D/9wkHLlKwCwJo13xlsvjyMpdWavISAQP72BpR+OommX3hQvXzndsudPHaVMtZqobheKTkFRdCTavZt9/M+1q/j5+xEUlFrHdauW06FLNPZ4e5r6ZC2u0+lk8ltDGDJ8PAULa7vl094ZRqee/VP2ZpI5fewwVWvUxu3y5K/TYY/3Pn9/fz8K3JV/+87R2O12FEWHouiwZWNWto+PDxM+mMHStd/io/OhWctQANavXk67TtGaVygl/qPpFbofRw//wBO16uByu1B0OnRK1r1O9Ro05OyZUwCcPnWCMuXKAbB21TI6dolJ773KZn0ynOyoKMpEMpjorKrq2PvErQrUVhRlCBCiKMrLqqrO8Ta5dffwFtX1eIs+9niLenm8RUuz4S36/uABPpg0Dr2/ngZPN6ZuvQa4XC72pPHO9PF4Z77Oonfm7KG97Fq2GIAipcphMAXw095t3Lp2mT0rP6dFTD9qN48A4LuNy3nmrckoio7Nn83kzPd76Dd+llfvYduWjUREtU/52+VysW/3TmZ+vAhzXBz9e8WgqipfLF2dpXh7t67nl1PH+WTqBAAGvf4O+3Zs4uqVv1j11UKi+wyiabjm0du8aglvvDsdRafji7kfcWhvLGOmLvQq/+1bNtI6Mn3++/fsZMZ8Lf8Bz2r5f/6frOWflqt/X+bVwf1QdDq6du9F8RKlcLlcHNi7i2lzP8VsjmNg726gqiz6ZpXX8UXhcDjo36MTP585Rd/uHXhz1HgWfzybI4cOcuH8bwwaOozwZK/TN58zKY3XaU/sNmYu+CJL66lRszYGg5HObcMICirI4Jde1bxLu3cy++PFmOPi6NsrGlVV+Wqp93umkMlVJEVRmmf0IlVV92Z5BYpyILtXkXLK/a4i5ZR7XUXKbbJ6FSm73H0VKbfJ6lWk7JLZVaTc4GFcRcpNMruKlBvc7ypShnswaZuIoigVgRKA1zfHyKy5SCSS/23uez8YRVHmAvmAxsBBwATsE5yXRCL5HyArJ3lrqqraG7ioquqzohOSSCT/O2SlwTgVRdEBdxRF6QNUEpyTRCL5HyErDaaXZ7kXgEJA9oecSiSS/1dk5Z68lT3/ARwB8opLRyKR/C+RlQbT0vN/BXgCbWyMPMkrkUjuy30bjKqq76b9W1GUdbmZgGgvklOQsgQ054w70S4svrYOsdqMJHu80PjxwqQ0nvg2K6pb3Drs8TZsOnHzlDSvU4Kw+PZ4Gzh8hMWPv88I36xcpu6f5s/iaOdhchVR6hmdAj9fteBvEHO/j6SEeDrWKEFAoJhBagAWs9Z8Awx+91kye4SU1I54Rb2H+kO+AMDmFDMrJcDXza4p0QQE6oXE90eMTygthQL0BAQahMS2KU4UnUKAUcz2o3NlHjcrh0jJ304VOA3MzFlK6RHtRfI3OPA3BgiLHxAYKNSZ4/b8OouqkeJp7qLeQ3JjsQr8FQ0IDBS6Del0itDPWHWrQvNXBOefGVlpMGGecTAAKIoyC3hVXEoSieR/hcwmOz4J1AGeSnOY5As8+SASk0gkjz+Z7cG4ASdwi9TDpASgr+CcJBLJ/wiZTXY8AZxQFOUPYL+qqqqiKArQBDj/oBKUSCSPL1k5tT9O9dzTwfP/cWJTkkgk/ytkpcGYFEXxh5QbeYu7JiuRSP6nyEqD+RDYryjKCrQRvJvFppTKj4cPEdqiCa1bNWeEx4vUrk04bSNSvUiDn/fOi/TLyaO81iuK13u3Z8EH7xB3+yav9Yrizec6Mu6l3iQm2HG73Ywb2odhvdpy7Yrm/Zk78W1sXnp/RPP3lSs0blCXgnmNOJ1OzGYzbSPCiIoIzXZ90nLlyhWerl+XAnlS40dFhBHZOjX+oAHexQ+vU5pt73Vg23sdOP9lH7o0rsCeKZ25vuJ5KhTXxuQoCqwY3YZdH3aiTGHt92zGkKbkNfl7lb/o+ojm7ytXaNSgLkF576p/mvwHPcL5Q9YuU28GkoBo4CpgvN8LFEUpBxwCfgaSVFVtnZ3kypQpy+atsRgMBgY89yzzZs+kT99+gKZc0BsMhIW3xscn62MsihQvxUefrcFfb+CDt4Zw49rfTF+yCZ1Ox9fzp3Bo7w6KlypLtZp1qFmvEfu3b+Spxi0pUqIUAXkerWlYBYKC2LQ1lp7dugCwa+cOnuvXP+WxQW8g1Mv6pCUoKIgt22LpEeOJH7uD5/r2T3mcnfrvOPYnO45pTXvf1C5sP/YnB878zaS+DVOWqV2hED/+9x/2n7pM58YV2XHsT/78x4I5Psmr/EXXRzQFgoLYfJ/8va3/gybDPRhFUbopirIEWAaUAsqqqtpRVdVRWYy9Q1XVFtltLpDei+Tj60vBQoVISEjAZrNhNJlYu3oVXb30IgUVLop/ivfHF19fP3Q6rQwul4uSZcqjNxhJSkwgwR6PwWhi/X8W06Fn/8zCPhQMBgMFCqT6c0wer5PNZsNkNLF2zSqic+CNuju+0WRK8VIl1z+6W/bilyuah2t37FjtDv65k366RXyiE4O/DyaDH7YEB0Pa1WDB5tM5zj+36yOaf9X/Lm/Xmkc8f8j8EGky2p7Le6qqLgQSM1n2XrRUFGW/oijDsp2dh2QvUr80XiSL2UybyCjGjhnJmGx4kc6fO4P5zi3KVqrKLyePMbRbOCcOHaBYqbKUqVgFl8vF7s1rKFW2IqXLV2bNV58w/71R2Kxi5k3lBi1Dwzh29AgnT/yE2WImIjKKsaNHMGbkW7ni/WmVJn5y/d8ZNYLRI7yP36lRBTZ8f++Lkef+uoOvTkf35pX59Uoc//3rDi93rMm0QY3Jk4Mh76LrI5pWoWEcT66/xVP/0SMY/Qjnn2GDUVW1MjALaK8oygagmqIoLT0neu/H30AVtJnYYYqi1Ez7pLdepDdee4X5CzQv0ozZ85g6Yzbbt20lIDCQ4OAQgoO98yKZ79xm3uSRDJswA4BqNeswd8UOGoVFsW2NZlvp/9po3pw8m91b1lCrXiPcbjdNW7dnz5bs3V39QeDn58fMOfOZNnMOO7ZtJTAgkOCQGgSH1GDPLu+9UfeKP2vOfKbPnJNaf0/83V7Gj6pfjs2HL2T4/Lglhxg8azfdm1dm36nL6HQKa747T7fmlTN8TVbyF1kf0aTNf/s2zduV3fo/KDI9yauq6klVVcerqtoBCAOeArbeL6iqqomqqtpUVXUCm4Aadz2fZS/SwH69mfR+ei/SyuVL6RrTLb23JYteJJfTyUcjXmTgm+MIKlwUR1Lqcb0pIA/+htRJZ2eOH6ZySC3cbjc6nQ5FUbz2/jwMVi5fStfobsSn8TpZc9H7s2JZ9usPUDS/kSSHi1uWzHeKG1YryrFf/8FHp+B2q6iqSmAuTPoUXR/RrPDknx1v14MmKyd5AVBV9TwwxfNfpiiKkkdV1eRjicaA104kgLWrV3L06BHGjta8SOMnTuapevXZtTOWTxZpXqSeMZoXafnqdVmKuW/bBv57+icWT58IQO+X3uKruR+iKDry5MvPWx/MS1l26+pveHXcVBSdjiXzpnB4fywjPlqQnbciBIfDQef2UZw+eYKObdswfuJk6tR9Kl19esR0RlVVVqxen634ndpHcerkCTp44tf1xF+4WIvfPVqLv3JN1uO3a1ieTYcupPz99dvhNKpenErF8zF9zU8pz/VtXZ2h8/bhVlVGP1OPNk+V5bmpsV7lL7I+orm7/u+myX/hY5A/ZOJFylFQRYkCJqKdtzmgqupbGS1bq/aT6neHxHmRjly8LWw2dZLdRoMKBYXOVI2L0y5Hipptm3xOSdR7KNNda9iiZlMH+rk4/WkfYfWxWixCZ1NbLBahs6mtFovQ2dQWi4UiQXm99yLlBFVVtwBbRMSWSCSPD6Lc1BKJRCIbjEQiEYdsMBKJRBiywUgkEmHIBiORSIQhG4xEIhGGbDASiUQYQsbBeINo8VpSgjixWFJCPDarGB9PMt4Mwc8O9xNn5ZQAX3Hiu+T4Imtks1rRiRJ3eeKLFMfZrFYUwflnxkNvMACKIqYACgq1SuUjIEDMTfhsNj9Utyp0A1l/5goAOv19b8OTLSIr5APA4RTTCPZMiwHEid1sVis95h0kATFiMQMOlr3USEjsZFyqikvQNuRSVXwRawfNjIfeYESL11RVrNRKpyhi43sai05vEhI/ufmKeg9JftoUAZHTKRLww656d7e7LKOIl+s5nW6h25Cvr+6hidfkORiJRCIM2WAkEokwZIORSCTCkA1GIpEIQzYYiUQijEe6wYj22ojwLt2df257ba6cP8f0F6KZ+VI3vnlvOJbbN5n+QjSzhvZg4YiBJCUm4Ha7WThyEDNeiOHW1csArJg+FrvVO6/T2TOnad2yCZFhzXlx0ADi4uLoEBVO+8jU+rwwqP8j5eUpWcDId2Na8tWg+iwe8BSKAlO612TJ4Pp8/vxTFDD5oSgwv8+TLH2hASXya7dIHduxOoGGh35RNR0/Hj5EWIsmRISmbp/tI8Np1ya1/kMGPtpeJCF3tANQFKUP8BzgA/RSVfXyvZarVftJ9eDhY/eMkZCQgN1up2e3Lmz6dgebNq4nKVG7j6u/Xo9Bb8BsMdOte897vt5qsWR6mfra1avky58/xbtUqXIVKlaqBIBer0dvMGAxm4nJJH5ml6nvlX+iJ399FvIH+HT/WSD1MrXL6cDHVxvz8c17w2ncsRdlqtdEp9Px7eezKFauMgVLlOaXQ/uo9GRDLpw5TrX6TTn7/R7Ceg3+V/wuVTQtxr3eg8PhwM9PW9eLgwZQpWpVSpYqDSTXX4/FYiG6W48M808e6CjyjmrhH+5OuUxdsoCR11pXZvjykwAEl8jLs43KMGrVadrXLk6BAH+OXrhNkyqFOHz+FrXL5OfAf2/QvFphFu3941/xjUoSO95uKTT/jC5Tp9s++3q2z4pZ3z5B20ZFXqa+3x3thOzBKIpSEmiuqmqox410z+ZyP0R7bUR4lzLLPze8NsnNBcDX35+gYiVTvE5ul5vCpcrhrzfiSEoiKSEef4OR/Wu+ommX3l7nn9xcQNugS5Uuk5K/yWhk3drVdInu5nVc0TSoGMQ3Q+rzXJOyXDMnkPwTmsfox534JOwOF3pfHUZ/H+xJLp5tVIZvDl56qDnfi7Tbp6+vLwULFiIxMYF4j5dq3ZqcbZ8PAlGHSBGAj6IoOxVFmaMoSq7ckFWU10aEd+le5JbX5tSBWN7v0wbL7ZsE5MvPxbMnmPJ8B349dpCCxUtRrFwl3C4nR3dsoEjp8hQpU5E9Kz5j1cx3sdu8m5axZdNGnn6qFtev/0PHzl35Kbk+FgsRbaIY/85Ixo7OnfrkBv+YE4iYsp8+C3+kUaWCFMqjx99Hx5Y3mtCzYWm2n77G+X9s+OgU2tUuzoUbNs5ft9G3aVlGd6hOgP7RsySm3T6PHTvKSc/2GdEminFjRvJOLm2fIhDVYIoC/qqqhgLxQMfcCCrCayPCu5SV/HPitXmiSRgjv9pK/kLFOH1wF2WDazF80QZqNovgh80rAWg/eDi9Rk3hyI4NVH6yIW63m9otIjkWu9GrnKPatef7IycoXqIkW7dsZtqsuXw0fRY7tm8lICCA6sE1qB5cg725UJ/cwOFSsTtcuNwqe365TpWigVgSHERNO8Dc2N8Y0Kw8ADO2/cqoladpV7s4h36/hU5R2HbqKu1ql3jI7yA9t27d4s1hrzDvY8/2OWseU6fP1uofGEj1kBCqh+TO9ikCUQ0mDtjrebwLqJ72ybTitVv3Ea/di9zy2ojwLmWFnHhtHEmpLiFDQCA6XeovrsEUiJ8+1et0/tRRylR7ArfbpXmddAqJ9qxP/kw+XwSQN08ejEYt9qoVy+jSNYZ4uz1N/R8NX1SAf2o96pQtQJlCJuLsDgBu2xzpTuQ+WTY/p/8ya94lVfMumfwfnT0Yp9PJwP69mfjev7fPLtHits/cRNRp84PAQM/j2kC6s2ceFe1C0E7yZhREtNdGhHfp7vxz22vz86F97F6+GIDCpcqRN6gws4b2QNHpCMibj95jpqcs+/2m5fQYPhlF0fHtZ7M488Menhs7M8v5x27fxrw5mv2yYsXKtAprrSl1d8Xy8cLPiIuLo1d3rT5LV67LclyR1C1fgFdbVybJ5ebYH7dZsOs805+pxVeD6qNTYOTKUynLxtQrxdg1Z3CrKkPDKtGsWmHeXHryIWafnrWrV3Ls6BHGjdG2z3ETtO1z965YFnyqbT/PdNPqv2zVuoebbAaIvIo0Fc0EeQN4RlXVpHstl9lVpJxyv6tIuRFf9GTHu68i5TaZXUXKDR70VaTc5mFeRcoNHvZVJGEX/lVVfVNUbIlE8njwSA+0k0gkjzeywUgkEmHIBiORSIQhG4xEIhGGbDASiUQYssFIJBJhyAYjkUiE8dBvgCHUi2S1oiLQOWOzohOshHAn2oXGt9nE6D6ScYjO32rFgANRH4MBh3DvktMlzh1ls1nx9RG3H/HIe5GcLjf/mBOExLbHJ5LH4IuvIOdPktPNu1vP4fQRJ1+bGF4OAJMgt9PN23EAuHQOIfH90WrvEvQlcrndLBpQT1h9RIvpACwJTpyC6m9PcKIokETi/RfOBvHx9xygn8JDbzA6nQ+mAHFD7U1GPwIDBTptfPQ4fAz3XzCbJH9xRL2H+ERtmr9JUHy9qm3YIqdTKH4uAgR+xgFGP7FeJzVRWP0BdApC65Ppuh/KWiUSyf8LZIORSCTCkA1GIpEIQzYYiUQiDNlgJBKJMB65BnPt6hU6hD1NcJkCOJ1OHA4H0VEtqFm+MBf++B3Qxs4M7hNDt3atuPyndjf4cSNew2KO82pdly5eoHqFknSIDCW6YyQWs5nO7VrTqW04Fo93Zujg7Hl/OjxRlPfbV6N8kJFJbasyqW1VFvaoSfsaRVGAUeGV+KB9NQoHajdKGtyoDCa/7N2ucf6cGbQNb56r+YN2y8ahz/emR8cI3h8/CovFzDOdI+nZqQ0Wixb/jaEDH1kvFcCKpUuIbh9B57Zh/HnpItHtI+jarnVKfV55YcAj6xVKrn/3jq15b/woLGYzPTu1oUfHiJT833jp+Wznb4+Pp0/3TkS3D6d/r2jM5ji6d2pDtzTxh+UgPgi6TK0oShtghOfPqsALqqquy8pr8+cPYsmqLbzQT3Pt+Pr6suCL5Xw06Z2UZc6e+oladerRsHEzvt20lmYtwihZqgx58ubzOtfmrUJZsOgrADauX0OvPv0A2LM7Fr3eQMuw1vj4ePfF99UplA/S7kD3xy07YzafA7Sm8uOlO1QoZOK/122cvmKhUfkCHP/LzD/WJOId3n+QiYmJnDl1MiXn3Mg/mW2b11O9Rk1eem0440YMY/HHc+je6zkADuzZiV5voFnLsGzHL1OmLJu3xqZ4qebNnkmfvlr+u3fuQG8wEBae/fz/vnKZ7w/sZ9XGbQBsWr+GZ/r0BWDv7lj0BgMtQ8OzHV802zavp3rIE8wdtoSxbw9j8YLZdH+2LwD79+xEbzDQrFX289+9cztP1q3HsLdGM3vaB2zbvIEenvj79uzEYDDQPAfxQdAejKqqWz0+pBbAJSA2q6/VGwzky5/qElIUhUJFiqZbxmA0keTxw5hMJpZ8toBn+/1bKpYVDuzbS7vWLfh47kyMRhOJabxFG9auonNX770/rasVZtevN9K/L18dBUx+XDUnkuh04++joPfTkeh00zakCFvO/pOt/L/+cjHde/UByLX8k7l08Q+qB9cAILhGLQoWKkxiYiLx8fEYjCY2b1hD+84x2Y4v2ku1e+cOXG4X0e0jGDX8NfQGAwkJicTb4jGaTGxct5qOXR49r1Myly78QbWQJwAIfqImBQsVITExAbvHi7Rl/eoc1b9c+QokeW4iHxcXx+W//ky3/Wxav5oOOYgPgg+RFEWpAFxTVTVXh0NWqlINp9PJxjXLKV+hMhUqVeXzT+YwYdQbKbvuWaFoseIcOn6WdVti2bt7F8WKF+en40c5feoEFouZ8DZRTBg7ivFjRmTZO+OjKIQUz8OpK+mnP9QtnY9jf2qHcH/dSUCnKDSvGMTluAT+upNAhyeKMvDpMhj9sv6ROBwODh7YT7PmLQFo3jI0x/mnpUKlKhw6uB+A7w/s5fatm5z86RhnT5/EajHTKrwNH04Yw/vjRz2SXqob/1zDkZTEqo3bMBqN2KxWTiTXx2wmrHUkk8aNYsI72auPaNLVf/9ebt26wcnjxzhzylP/1pF8MGEM72Wz/uUrVOL40R9p9fSTnPzpKC+88ka6zzc0PJL33h3DpHHZ/3xFn4PpAqwVEfjN0RP4cNZCNqxZToPGTXGrbtq078ymtSuyHEOv1xMQEICvry+tI6P45eezTJkxlw+mzmLnds1bVD04hOrBIezbsytLMVtULsi+3/6tYmlYrgDfX7id8vfXRy4ze98FmlcsyOkrZnQofPfHLZpVLJjl/Fcs/ZquManaVj8/vxznn5awiLYkJNh5pnMk/no9xYqXYNKUWbz7wXT27NyOKSCQKtVDqFI9hO/27fY6Poj1UuXJm4+nGzcDoEmzllw4/zsfTp/De1Nmsit2GwGBgVQLDqFacAj7s1Ef0YS1aUuC3U7PTm1S6j956mwmfDiDPbHbMAUEUrVaMFWrBWer/iuXfU3zluHs+v44oeGRbFy7kvemzmbihzPYvVOLX616MNWqB3Ngb/Y+X9ENpj2w4e5/TOtFun3Ley9SMsd+/IEnatfB7XKhU5L9Qln381jSTLI8/MNBypevAMDqlcvo1FXzFik6HYoX3qKS+Q1EBhdhXJsqlClgpG1IEXwUhVL5DVy4lX7iX7Uigfx2w4ZOp+BGRVXB4MUezG+//pfPF39Ct85t+eWXs3y6YG6O80+Lj48P734wg/+s/RYfnQ9NW4YCsGH1ctp1iibBHq/VXdFla86OaC9VvQYNOXtG05ScPnWCMuXKAbB21TI6dolJH/8BzDnyFh8fHyZ8OIOl67bi4+NDsxZhAKxfvZx2nbX8Uz7fbNRHVVXye9TGQQULppxYX7d6Oe07xeR4+wGBc5EURSkGJKmq+q8OktaLFPJE7XTTnR0OBwN6duKXM6fo170Db4waz+IFszl66CAXzv/GwJeGER7ZHoAVX3/OxKlz0el0zJoymT07tzHj4y+ynOMPBw/w/qRx6P31NHi6MXXrNcDlcrF3VyxzP/kMc1wcvXt2RVVVvlmetR2xrw7/lfL4/fbV2HzmH54slZeTV/49Yzy8WiHm77+IikqPOiV5qnR+pu36Pcv5j5v4fsrjtuHNGThkaI7zT8vVvy/z6uB+6HQ6unTvRfESpXC5XOzfu4tpcz/FbI5jUO9uqKrKom9WeR1ftJeqRs3aGAxGOrcNIyioIINfelWrz+6dzP54Mea4OPr2ikZVVb5ausbr+KK5euUyrw7pi6Lo6Nq9F8VLavU/sGcn0+YtwmyOY2DvGFBVFn292uv4naK782L/Z1m94j/4+fox/7Ovtc93z05meOIPeDYGVVX5/Bvv44NYL9JgwE9V1bmZLRfyRG11w87vheQQb7OQV+BkR6vVwhvrTgud7Di7fSVA3GTHf25qh22P62RHq8VCfJK4yY42q4U8Aic7WiwWblrETXaMt1qETna0WS1ULVP4oXiRPhEVWyKRPB48cgPtJBLJ/w6ywUgkEmHIBiORSIQhG4xEIhGGbDASiUQYssFIJBJhyAYjkUiE8dCtAm63i3ibGC+SPd6Gr1vcW4y3WfF1idFBpF2HSOzxWZ9akR2cZK61yCk2mxV7krj7ucTbrOhc4txRNqsV+33UHznBHm9DEajuir/P9vPQG4zTrfJ3nBgvUpI9kQIB/vgJEk/5+uh4rXlFjKYAIfEB/Dy+HH9B78Hh1kZy+7rFjOjWecR3DkHxnW4VXx8FXx8x3yJRcdPy6eFLuH3FjAbXORMY1qwCAQYxTVJxZd5CHnqDURQdeqMYaRZAQECgUCdPvDtRmPQLIMDTYES9B6NV2wMT5abSe4RfIt1UqqoKjW/09xXqRXL7GnD7GYXFF/0dyAx5DkYikQhDNhiJRCIM2WAkEokwZIORSCTCkA1GIpEI45FrMGdPHOGlHm14uVdb5r4/GqfDwYvdI2hTpwx/XTwPaF6k0S/2YmjPSK5e/hOAmROGY/Xiht8ARw4fIrxFE9qENmekx8vTPjKcdm1SvTwvDPTOy3Pt6t90Cm9EjbJBOJ1O/rx4gZ4dw3mmU2tef6EvLpcLt9vNC891o0f70BSv0/gRw7z2Ov14+BBhLZoQEZrqFbo7/yFe53+FjmFPE5LGSxUT1YJa5QtzMY2XakifGLqn8VKNz4aXKpl5s2cQGdYMs9lMp6hwOkam5v/SoOx7nZKZP2cGUeHNtfhtW9MxKjw1fg68USIJq1yQt1qWx0eBt1tVYHbn6hQO0BxaCvBiozK81bI8QSbt8nPPJ4tj9PXu6yxi+78bUV4kE7ASCADigG6qqmZpRFrREqWZ/uU69HoDk94czKXzvzJp3hI+mfpuyjK/nj1J9Zp1qV2/CXu3baBek1YULVGawDx5vcqzdJmybPR4eZ7v+yzz58ykz3OpXh6DwUCol16h/PkL8OXKzbzUrycAefPl45OvVpI3X36mvz+evTu3UaRocWrWqUeDRk3ZumktTVuEUbJUaa+9TmXKlGVTsleo77PMmzOT3s+l9wp5n38QX63awotpvFQfZ+ClatC4WUr+JbLppUrnddoVy7PP9U95bDDoaZUDr1Ny/NPp4vdLE99Aq9CcxReBr067hzOAS4X5312ka83UexaXzm/gj1t2zl23UrdUXs5ctXIr3oHd6fZqPSK2/7sRtQfTBjjk8SId9vydJQoWLope73Hl+Pig89ERVKhIumUMRiNJSYkk2G0YTCbWfrOITs8M8DrJtF4eX19fChYsREJiAjaPb2ndmlV08dLLc7fXKV/+AuTNl9/zfnzR6XRa/gma38ZkCuDrzz+hVza8TvfKP9HjizJ68vfWK5RVL1Xa9eTES7Xki8X06NUbAKPRqHmR4m0YjUbWr1lN5+iceYuWfLGYHs9o3iiTKdUbZTKZWL92VY7ji6BJ+QJ8f+FOyt+WxPR7EEkuFT8fBb2P5tVqWakgu+9hsrgfIrb/uxHVYH4H9J7H+QGv3/3v584Qd+cW5SpV+9dzZStWxeV0ErtpNaXLVaRMhcqs/PJjZk8agc3q3WESpHp5+qbx8pjNZlq3iWLcmJG8k00vT1quXf2b7/fvpkmLMM3r5HKycc0KylWsRIVKVfhi4Vwmjn7T68O8tPn3GzCQY8eOctLjFYrIxfzTUqlKNVy54KVyOBx8d2AfzVq0AqBFqzDN63TyBBaLhfA2Ubz7zkjGjc5e/ineqBap3qi03qXWETnzRonAR4EqhQM4dz3jIfhXLYnoFIX6ZfJzzZLEVUsiYVUK0b12cQxeHiaB2O1fVIP5FWigKMoZ4CngoDcvNt+5zayJbzN80qwMlxn4+juMeH8usZtWU7t+Y1S3SvOIDuzc5N3d4W/dusXwYa8w92PNyzN91jymTJ9N7PatBAYGEhwSQnBI9rw8ySQlJvL2KwOZNHUuvr7aUekbo97lg1mfsHHNCuo3aorb7aZNu05sWrvS6/zfHPYK8zz5z5g1j6nTZ7Nju+YVqh4SQvUc5n8v3vB4qTbmwEu1/D9fE92tZ8rffn5+TJ05lw+nJXudAqgeXIPqwTXYm438Vyz9mq7d0nujkuPH7tiazouUHW+UCBqWzc/hS3fuu9y609f48shl6pfJx7l/bCjAsb/iqF/Gu8NU0du/qAbzHLBNVdUQYDPwbNon03qR4u7cSvdCp9PJ5LeGMGT4eAoWTr9rfjenjx2mao3auF0uFJ2CotN5NXnP6XQyqH9vJr6X3suzavlSukR3Iz6HXp5kxrw5lGf6DqJS1erp/v3Yjz/wRK06uF1udNnwzzidTgbeI/+Vnvxz6hW6H8d+/IEad3mp4r3wUv326zk++3QB0R016d3Cjz1epxXL6NQ1BrvdnsZb5P2kzF9//S+fL/qEmE5t08dfuYzOXdN/vlYB9ckORfPoaV4xiFealqV4XgMtKwVluGyFgkYu3rajU0AFVFVTFGeVB7H9i5qLpADJneMGkK6tpvUiVQmumW4W3N6t6/nl1HE+mToBgEGvv8PKLz/m1NEfuHzxPD2ef5kmoVEAbF61hDfenY6i0/HF3I84tDeWMVMXZjnJdatXcuzoEcaN0bw84yZMpm69+uzeFcvHn2penl7dNC/P0lXrshTT4XDw/DOd+OXsKfr36MDQ10eyY8sGrvz1J199Op8+A1+kdVQHAFZ+8wUTpsxBp9MxZ+pk9u7cxrT5n2c5/7X3yP8pT/4LPPk/48l/mTf5e7xU/bt34PVR4/nM46W66PFShd3DSzV7ipb/dC+8VOMnfZDyODKsGYNe0LxOe3bFMm+h5nV6toeW/zcrspZ/uvhpvFFR4c3Tx0/2RvXQvFFfrxAiIPWaNaeupTx+q2V5dv92i0ENS1OpkIkigf5sO3eDEx7HVtPyQXx99DJuFdqHFOGJYnlYdOjPLK9LxPZ/N0K8SIqi5AeWo52HcQDdVVW9da9lqwTXVD9duyfXcwBItFupUTKfUCfPdYvYyY6Bgic7/vH3DeDxnexotVqETna0Wi1CJztaLBbe+fYXYZMddQ47Y8OrCP0OlCyS/8F6kVRVvQNEiIgtkUgeHx65gXYSieR/B9lgJBKJMGSDkUgkwpANRiKRCEM2GIlEIgzZYCQSiTBkg5FIJMJ46FYBVXWTaBczTDvJHo/NJm4qvs0m1mkDoHgG2olCtBfJJdiLFG+zImKwaNr47iRxXxOb1YrOKUbbA5q2xJvpJ95yv9gPvcHofXVUKSpmJGy8DW5YkrAJkqPZ45PIb/LFZBIn5nIkiG0wAf5aAzbpxWwK7iQtf19B9i8fRUGn06H3FfND4nwA94rp91QpYW4te7yN/p8dJhF/IfH19/kBeegNRqfTESDQaZOkODEKHMpvCvAT6uRJ8gjXRA31tju0e42I+gzcidoXVKSXR6dThMcX6UWyunyETdUASMSfhJS7pzxY5DkYiUQiDNlgJBKJMGSDkUgkwpANRiKRCEM2GIlEIoxHusFcuniB4Iql6BQVRkzHKCxmM13bR9ClXWssHm/Ly0MGeO0t6hLeiJrlNG/RX5cu0KtjOM92as0bL6Z6i17s242eHVK9RRNGeu8tunTxAtUrlKRDZCjRHSOxmM10bteaTm3DU/IfmgMvjwgv0t2sWLqE6PYRdG4bxp+XLhLdPoKuaer/ygve1f9B5v/3lSs0blCXgnmNOJ1OzGYzbSPCiIoITYk/+Pmc1UcE165eoUPo01QvXSDlZtsL586gW7tQhg3ph8PhwO12M7hPDDFtU91U497OmpuqRH4Du99qyuJ+dVnQ50kAGlYMYlHfOizuV5fqxfOgKDCrZy2+fP4piufTzAOj2lYlUO/dZXtRXiRf4GugKPCjqqpvZTdW85ahfLzoSwA2rl/DM737ArBndywGvYGWoeFee4s+X7mZof21m03nyZuPjz3eohlpvUVP1qN+o6Zs27yWJi3CKJENbxFA81ahLFj0VUr+vfr0S8lfrzfQMgfeGRFepLT8feUy3x/Yz6qN2wDYtH4Nz/TpC8De3bHoDd7X/0HmXyAoiE1bY+nZrQsAu3bu4Ll+/VMeG/QGQsMfPS9S/vxBLFm9hRf6ajcsv3njOj98t5cVm3byyexp7Ph2I6XLltPcVI2a8e3GtTRt6Z2b6vvfbzFq9RlAG4sW81RJBn15DLdnzGL14nk4dTmOH/+4TXhIEQ7+dpO/4xKwJnrXjEXtwXQGTqiq2hIwKopSK7uBvtu/l/YRLVkwdxYmo4nExETi4+MxGU1sWLeaTl2989pk6i3y9cXHR4fRaCQp0eMtMgbwzWef8Ew2vT8H9u2lXesWfDx3JkZjqpfHaDSxYe0qOnuZf1pEeJHSsnvnDlxuF9HtIxg1/DX0BgMJCYnE2+IxmkxsXLeajl0e3fwNBgMFCqR+1iajSfMu2WyYjCbWrllFdA69PyK4exs9efwIDRo1A6BR85b8dORwyrZk9ziMlixeQO/+Wd9G65UvwBcD6vLs02WoVTofbhXm936SyV1CMPrpSHC48PfVYfT3wZ7kokeD0izz4n6/yYhqMBWAk57HPwFPZydI0WLF+f7YGdZu3sG+PbsoWrw4Px1L9uaYCY+IZOK4Ubz7Ts69NsneosbNw6hYpRpOp5NNa1O9RV9+MpdJY7zzFhUtVpxDx8+ybksse3fvoljx4pr355Qn/za54+UR5UW68c81HElJrNq4DaPRiM1q5URy/mYzYa0jmTRuFBNyWP8H5XVqGRrGsaNHOHniJ8wWMxGRUYwdPYIxI996ZLxI98IcF5cy0C9PnnzExd3W3FQuJxvWLKdcxcpUrFyVzz6Zw7tZcFNdtyTSYdZBBnx+jIYVg6hTNj+F8/jz4pLjnPjzDtH1SvHHjXh8dQpRTxTj4s14Ltyw8WyjsrwdVYUALw6TRDWYc0Bzz+OWQIFMls0QvV5PQEAAvr6+hLeJ4tzPZ/loxhzenzqTnTu2ERAQSLXqIVSrnjOvTVJiIiNfHcjEKaneotdHvcv7Mz9h01qPt0h1E9HWO29R2vxbR2pqjikz5vLB1GTvTyDVg0OongMvj0gvUp68+Xi6sfbL2aRZSy6c/50Pp8/hvSkz2RW7LZ1XaP8jmP/d+Pn5MXPOfKbNnMOObVsJDAgkOKQGwSE12LMrd71RuUnefPmwWDSTgNVqJm/e/AC8OXoCH832uKkaNUV1u4ls35mNazJ3UzlcKnaHG5dbZd+561y6Gc+xi3dwq3D4/G0qFNKmLcyO/Z2x684SVbMYP/5xG50CsWf+IfKJYpnGT4uoBrMR7dBoJ5AIXEv7ZFov0s2bGUsfrZ6iAhz+4SDlylcEYM3KZXTqEkO8PY23JQcTut4ZPpSez93bW1SjZh1cLneq9yc+6+ux3JV/+fIVAM3L06lrN+z2eJRs+JCSEe1FqtegIWfPnALg9KkTlClXDoC1q5bRsUtM+viPYP4ZsXL5UrpGd0u3/VgFTgjMKU/Ursvh7/cD8N3e3dR+qn7Kc0cPa24tl9uFotOhU+7vpjL5p+6B1C6Tnz9v26lQWGsqVYvn4fIde8rztUrn48xlMzpFwa2CW1XTvf5+iLIKuICXARRFWQhsv+v5FC9SzVq1M5wK+8PBA3wweTz+/noaPN2IuvXq43K52Lt7J3MWLMYcF8dzz0SjqipLlmXN6OhwOBjUqxPnzp7i+Z4deHFYGm/Rovn0ef5Fwj3eolX/+YJ3P9K8RXOnTWbvrm1M9cJb9MPBA7w/aRx6fz0Nnm5M3XoNtPx3xTI32cvTU/PyfLPcey+PCC9SWmrUrI3BYKRz2zCCggoy+KVXU+o/+2Ot/n17afX/aql3Rs0Hkb/D4aBz+yhOnzxBx7ZtGD9xMnXqPsWunbF8skiL3yOmM6qqsmL1eq/ji8LhcNC/Ryd+PnOKvt078Oao8dRv2IRu7UIpUao0/QYPTVl2xTefMymNm2pP7DZmLvgi0/h1yuZnaGhFkpxujl+6w6m/zBy5cIfP+9clweHi7VWnU5btUrckEzf8jFtVebFVBZpWLsiINM/fD1FepJLAN4Ab+EpV1S8yWrZmrdrqzgM/5noOADarhTvx4iY72m1WCoie7GjXfo1ETea7fusOIHKyYzwgLn+rxSJ0smNyfJFepGtxdmGTHeNtFvp/9qOwyY4GEvlxYrsH7kW6DLQQEVsikTw+PNID7SQSyeONbDASiUQYssFIJBJhyAYjkUiEIRuMRCIRhmwwEolEGLLBSCQSYTx0q4Db7cZmtdx/wWwQb7Nit4u714c93oZecAkdCfFC48cLHiLvTrLff6EcYLNZ0QlSooDHW6QTG98eL86LpG2j4txU94stZCSvNyiKch246MVLCgE3BKXzIOI/iHXI+DL+g4xfVlXVwvd64qE3GG9RFOVIRsOSH4f4D2IdMr6M/6jEl+dgJBKJMGSDkUgkwngcG8zCxzz+g1iHjC/jPxLxH7tzMBKJ5PHhcdyDkUgkjwmywUi8QlEEDjqRZInH6TN4rBqMoijei4mytx4hdRGdv6IoQQJjK4qiVFBVVRW1gSuKolMUpYKI2GniNxAcv5+iKP6C4iuKopQA/ETE96wjb27GeywajKewXwDzFEXpoyhK/fu9Jhvr0CmKMhlAVVV3bn6JPLEXAHMVReme2xugJ/5nwEJFUZ7JzdhpGA6sUxQlWEST8TT15UCn3IybJr4CbAEaC4qvA1YCM4C6guJ/DUwHBiqKUiq343u20RmebbRObsR9LBoM0Aew4bmRONBGUZTIXF7HF8CbiqLMBcjlL9FnwC3gQ6A10CiX4ibzOfAPMAJooihK5VyOD3Ae7T3MUxSlgac+ubn9rAYsqqpOVxSls6IojRVFCcjF+PWBfZ74byqK0j63vkQeFgGHgA7AaEVRyuZibIAJgBl4FigGFM/l+G8BDmAUkBfopShKjrfTx6XBHPP83w2sAn4AaubyhzhJVVU9EO8xIaDmwiU2j0b3G1VVR6mqehpYC3TJadw08XXAXFVVR6iq+hvaMO/xiqIMVRSlSG6tB9gETAHeBz5UFKUuUCYX43+Kprr5Fa0ZvAg8pyhKtpxa9+AvoJKiKGvQDjGCgfaKouR4j0ZRFBPaZ/yRqqr7gJ1AJc9zufUjdQC4DBiBUsAoRVFeUxQlNJfiHwQuoP2I/AcojfZjlaO7kT8uDeZXz3/NAR+0YgQAFXNxHb8DeDzaKU1GUZQyiqIYsxtUVVUnsMdzmOeLtqEX8MSulpPYnvhu4IgnXi20PZkZaF7w3BxObgCi0Uyd76DtcfTLreCqqm4BvgRmqao6EhiJ9vnm1uHGFbQ6udD2+KYBf6B9WXOEqqrxqqqmNbfdQtvTyJUfKQ+HgTPAfKA82t78FTSLam7wM1rzegGIBAKBKuTwR+SxaDCqqiYA64DaQHu0WeD/AC09X9wc/0qoqupKjqOq6mvAFUVRDqF9oIYcxnaoGk7gFPCzoihdgJloH2qOSLMR/wy8oqrqEeA2EJpbhzGqqt4BPkazRXRGUwPXURTFJ7d+pVVV3QrM8zy+hDbhrl4uxVbRtqHLQAwQBOiB8Nx8D551fQkkKIoyKhdj3lJVdS0wF4hVVfUvIB/a6QK/nOavqup1YDHwN9AA7cdjF1qTyVHgx+Y/oCTa+ZglwLdAFQHrSB58GI42y7t6bsZGO779G+3XtKqgOkUD3wGVczluUeAr4F3P374CP+sewAmgYi7HLYG2d/EZWsOpJmj7eQp4AzAI+Awmou2l7s3t/NOsp5Wn/uVyEuexHMnruRyrU1VV2JR1RVHKAz6qdl4jt2O/BaxTVfW/AmLrgeeAPYLiF0quu6IoPqpm8cztdRiAXsB3qqr+ktvxPesogLYNZewuzln8fGjbzy0BsfOineRNUFXVm1udeLOOBsDtnG5Dj2WDedwR9cVME1+naudmhKEoiqIK3HgexHuQiEc2GIlEIozH4iSvRCJ5PJENRiKRCEM2GIlEIgzZYCT3RFGUFoqiXFQUZY+iKOs9V3ay+trxntfXVhRlQCbxszRILDleVtcveXSQDUaSGUtUVW2BNnI6Grybaa6q6k+qqi7O4OkW5N4oVMkjykP3IkkeC34C1iuK0hCooihKR7SBakWB62gD1/KgTR+wA0lo0yNaAGGqqo5RFOUFtPE5dmAI0BforChKLDAmK/HEv01JbiP3YCRZoRmQiDbwrTXwPLBBVdVWaF/8aM+/LVJVNYq7pj94Jl3GAI1VVW2JNq/sC+ANVVXf8Dae5PFB7sFIMqO3Z7bxWWADcNTz79WBuoqiDEabp7UU7XBnk+f5Y3fFKQ8cSx5cqGr320n7vLfxJI8Jcg9GkhlLVFVtqarqS2izkJNH1p4DPlJVtYWqqg3RJoT+AdTyPP/kXXHOA08mn7/x/N+BNjM+O/EkjwmywUiyw0K08yc7FUXZBdRBu+HSYEVRvkU7nEpB1WbqrgYOKoqyG6iMdig0SlGUsd7Gkzw+yKkCEolEGHIPRiKRCEM2GIlEIgzZYCQSiTBkg5FIJMKQDUYikQhDNhiJRCIM2WAkEokwZIORSCTC+D/GDLObMM8hhgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_cm(model1(torch.FloatTensor(val_sp).to(device)).argmax(dim=1).detach().cpu(), torch.LongTensor(val_label), 'model1.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4a88800d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAAEdCAYAAADXQ3g/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABdrklEQVR4nO2dd3xTVf+An5O2JB2UUvbeq+wNgoxOWja0gIqyFFScr8rLEpEhDpQ99FVxoLLKBhktSxRB9nIjqCxlNWnSleT+/rhpWpDRtDkMf+f5fPjQNDffe+65J9/ee894hKZpKBQKhQwMd7oACoXi34tKMAqFQhoqwSgUCmmoBKNQKKShEoxCoZCGSjAKhUIavne6AMagIlpgsTJSYgs0ygQb8fHxkRLf4XAghMDHIC9PWzOyABBCzj6MvgIAg6RjcDgcUuM7nU6cGghJ8TWnE1+DwCCpDTkdDhxOTWr9GAxCavyjRw5f0DStxPXev+MJJrBYGTqM+0xKbF9HOtN61CWocGEp8VMtFvx8DRSWFB9gx7HfATD6B0mJX7uY3gRk1dGFS1cACAySE9+aasGa6SAgUE58m9VCyWCTtHNssVi4Ys2UWj+FTb5SvwNlSoScutH76hZJoVBIQyUYhUIhDZVgFAqFNFSCUSgU0lAJRqFQSEMlGIVCIY27NsF0rVeSyZ1rIYBn21dmYlxNXulUg8JGHwQwMrIar3WpRYmgQgA81roCAX6eHc7xY0eJ7tiW2Mj2PDl0CCkpKXSLi6JrbCRmsxmAJ4YOdo/l8JQzZ87QukVTihb2x263YzabiYuJJDY6wh1/6JBBHsU/fmgfT/WL5ZmHujBnyhhSLl/kqX6xPNu/K2OeeIiM9DScTidjnuzPUw/Ece70HwBMnzCCVIvZo/KfPXOGtq2aUbxIgLv8XTpF0Tkmp36GPepZ+XPz+6mThFUrT4+4SBK6x2Exm+ndNYZeXaKxuOI//fgQj+OfP3eGruGtqV0+BLvdDsB7s98hoXM4zz0+kKysLJxOJ0MfTiAhriOn/9B7WceNeA6zOSVfxyKDrUkbie8SRXyXKBrXrsTSRQvp0z2GhG4x7vp57slH813/3+3ZTUSHtkSHt2fkS//x+vkFieNghBDTgGbAfk3TnvXks74GQeXQAACqFPPH7tR4ef1PtKsWSrvqxfj+XCo//23l6FkLrSuHcPC0mb9TM7FlOT0qY42atdi0dScATw4dwoL33+XhAYMB2LolCZPRSERkdL4H6oWGhrJ+YxL9EnoBsCVpMwMGDnb/bDSZiIzyLH7psuV55+MVFDKamPTiMP4+f5aZn6/DYDDw8ew32bVtE2XKV6JOg6Y0atGGHRtX07xtOKXKlieocLBH5S8aGsqaLzfzYJ/eAGxN3swjAwe5f85P+a+lfccI5r3/MQBrVi3nwYcHArBtaxImo4mOEVEexw8JCWXh8vU8PqAvABcv/M2unTtYum4L82dOZfP61VSoXIWGTZrTqs39fLlmBfd3jKJchQoEBxfJ97F4m46RMXSMjAGgS2RbNE3jgf56/e/YlozRaKRDuOf1k03FipVYtyEJk8nEkAH9mTNzutfPr5QrGCFEEyBQ07T7gUJCiOaefD6yVnG2/nwRgIvWLLLXxAoo5IMl3U6G3Ymfj8DkayDd7iS2Tkm+/P5vj8vp5+fn/tloNFK+QkXS09OxWa0E+PuzckUiveL7eBw3G5PJRNGiRd2v/QMCSM9Ix2qz4h8QwIrEZcT36etRzNASpShkNAHg4+OLr6+fe5Smw+mgfKWqmPwDyMxMJz3NhikggBWfvU+PB4d4p/zp6VitOeXvneBZ+a/l66+20zWmI/NnzyDAP4CMjAxsNhsB/gGsXplIj96e17/RZKJISE65D+3fS6s29wPQpn04B/btwd8/gMyMdGw2G/4BgXzywXweHvx4gY5FFqdOnqB4iZIUL16CjIx0bDYr/v7+rF21nG69EvIdt1Tp0phMrrbk60ux4sW9fn5l3SK1BpJcPycBrfL6QR8BdUsHcfSsBQBLhh0/H8HM3mF0qlOCb09e5nRKOj5CcH+1UM6kZHA6JZ2u9UoypFUF/D28TVq/dg2tmzXk77//onvP3hzcv48jhw9hsViI6RTH+JdHMW7Mf92X2gUhPCKS/fv2cvjQQSxmM51i43h59EjGjBzhcfxffzxGypWLVK5ei+8P72dY7wgOfLuT0uUrUalaTRx2B8lrEylfuRoVq9Zg2cfzmTlpFNZUS77L3zE8kgPZ9eMq/7ixoxg7On/1U6p0GXbtP8aKdZvZsW0LpcqU4eD+fRw9fAiLxUxUTCwTXxnNqy+PLFD9m81X3FdvhQsXIeXKFarXrI3dbmd14mKqVKtBteo1+XD+LF4d9R8sHt5KyubLNSuJ7dKd+ztEcOjAPo4d0dtnRHQsr40fw6RxowpUP0ePHObixQsMGvKYV88vyEswIUD2WUoBiuZ+UwgxVAixVwixNyP1ylUfbF+9GF+duOR+3bBcMLZMB88kHmfx/jN0r18agM/2nWH2V6doVy2Uo2ctCAS7Tl7m/qqhHhU0rktXdu09RJmy5diwfh1vz5jNm+/MYPOmDQQGBlInrB51wuqxfWuyh1XwT/z8/Jgxay7vTJ/Fpo0bCAwKIqxuPcLq1mPrlrzHN1+5zMyJ/2XEpBkA1GnQhHcTk7k/Mo4vE/VpF4/9Zyz/nTKL5LWJNGrRBqfTSfuYriSvTSxQ+afNnMPUaTNzyh9Wl7CwumzLR/0YjUYCAwPx9fUlqlMcP35/nDenzWLK1Okkb95IYGAQtevUpXaduuzYtiXf5Q4ODnE/f0q1mAkuot8GvTR2Am/Neo/ViYtp1aYdTs1JbNderFm+JN/7ksHmDeuJiu2Cn58fU96exaQ3p7M1aSOBgYHUCqtLrbC67Nyev/q5dOkSLzz3DHPnv+/18wvyEswVIPuGP9j12o2mae9pmtZM07RmxqCQqz5YLsRETO0SvBxTnQpFTZQtYiI1Q3/IZE63E1go536wVslAfrlgxSAEGhpOTcPkwRVMRkaG++fgwoXx99cvF5ctWUSv3gnY0tIwGAwYDAZSrdY8x70VSxZ9Qe+EPqTZbO741tTUPH3WYbfz2ognGPbSq4SWKEVWZqb7vYCgwhhdl7wAR/fvoVa9RjgcDvd+0m0FP46li/Nf/tykWnKupvZ8+w2Vq1QDYPnSRfTolYAtLVd8q+fxs2nQuCm7v9GftX29YwuNm7Zwv7dvzy7qN2qCw+nAIAwIg8BWgH15m7/On8OvkB+hocXcv1u5bDHdesaTlpaGQRgwCAPWfLRPu93OY4MeZtKUNyhVurT79946vyDvIe8uYBiwBIgEPsrrBz/97rT758mda/Hl8b94oWNVJsTVRAiYveOk+/3IWsWZv/MUGtC3SRmalC/CtG2/5bmQSZs2MmfWNACqVatBeGQ0DoeDrVuSmPfeh6SkpPBQ315omsYXS1fmOW42WVlZ9Ogax5HDh+jWuRPjJ06madNmbElO4r0PFpCSkkLf+J5omsbS5avyFHPbhlX8cOQA7019FYCBT/+XBTNfx2AwULhIUUa/Ode97bplC3nh1bcRrgfAu7cnMXbqux6Vv1e3zhw9cogeXWJ5ZcIkmrjK/+77evkfSNDrZ3HiSo/qBuDbb3by+uTxFCpkpGXr+2javAUOh4PtW5OZNf8DzCkpDHgwHk3T+HTRco/KPahfd74/doQBfbry0phXadG6DQmdwylbvgKDhj3t3nbxwo+Y/PZsDAYDM96czLbkDUyf/7HHxyKLjevXEBPb1f3a4XCwY1sy0+e+jzklhcH9E0DTWPC551emKxKXsm/fXsaNGQXA+ImTada8hdfOL4CQZRUQQswAmgCHNE176kbbhVaqranZ1DdGzaa+OWo29c25TbOp92ma1ux670vrpva0a1qhUPz7uGsH2ikUinsflWAUCoU0VIJRKBTSUAlGoVBIQyUYhUIhDZVgFAqFNO64VUCg4etIlxLb15FRoBGgt8JqTcXPR26OzkyzSY1vtcptArJHxdqsqaR5OIveE9JsVqyGgs9DuxHW1FRstsxbb5hPbNZUDA555/hW3687nmDKFjExvWc9KbGt1lSiBr6JNUPOYMJAo2DbpyMRQkiJDxBWVp9xERgoZ6CdzAQM4HBqV/0vI37RAD8CAwtJiW9F3pc/m2B/PwID/G69YT7wdfrhlDSYNk/7v2N7dmEwGKSNMgSwZmikpsv6C2cgMChI6kheh0Mvu8w6khnfmqH/9Zc1UhUg0ChvpCqAj4/c0dqapkmN73Rq0tvPjVDPYBQKhTRUglEoFNJQCUahUEhDJRiFQiENlWAUCoU0VIJRKBTSuKsTzN49u4nq0JZOEe0Z5fK2dI2NokunXN6ixzzztvib/Fg+83E2/u9ZlkwbSsnQwmz96D9sev9Zlk4fhsnohxCCJdOGsmXB81Qsoy8nPG1kH4KDTLeIfnuR7bWRHX9L0kZ6dY6iV+coGtaqxJIvFpLQLYb4rjnen2efyL/3R7bXSTYyvFq5uR1eJCkr2gkhygJrgTAgSNO0Gw6FbNiosfbVt/uu+975c+coEhKCyWTi0YH9qV6jJtWqVQegkNGIyWTCYjYT3/eB634+1WKhYbeXrxoH0z28IWHVyzDlvQ2MGBLD8V/PsG77UTRNY/TQWH44cY7fTl8gsnUdvtr3My0bVGHzN9/TqW1d3vk46ar4QSYDx9dPljqG4coVXQR2vXEMuetnyABX/VTX68doNGJ01U/CDeoHctbFlRX//IXLwK3HwcRFtGXgkGH4ulQyhQoZMZmMWCwWesbfWJ1hTbXccBxMeno6aWlpPNinN2u+3MS6Navc6zB7Uj8yx8FYLJYbjoPJLn+/hF6s27CZtauvX/4+/W5cfovFcsNxMN5qPzdb0U7WFcwlIAL4tiBBcntbfH19KVasuNsrFBAQwMrly+jlobflxJ8XMLoacUhhfy5dsZKdZH18DPzy+1/Y0jMxGf0IMBmxpmXyeL/2zF+8oyCHIgXZXpvb4c0B3ftTokRJipUoQUa6y/sT4M+alcvpXgDvz+3wOslEhlcrN/esF0nTtHRN0y57K162t2VgLm+L2WwmulMcr4wdxcseeFt++f0vmtevxL5lY2gSVpFdh36jWd1K7PxsBB2a1+TkmYv8+Nt5fH0M9I1txs+n/uKn387xdP+OvD0insKBd9dtEsj12tyO+Otd3p92HSI4dFD3/qRaLETGxDJ5/BgmFtD7k423vU63G296tXIj8/xKW/QbQAixDYi89hZJCDEUGApQrnyFpsd/PnnDGJcuXeKhPr34aOEit1rB6XQyfNgQesX34cIF3ehYomQpIqNirvrs9W6RHktoS5C/kWmfJPPcwxH8ddnC52v3APDsw+E4nRqzPtvq3n7uuAeZv2g7nTvUZ8fen6ldpTQfJH4N3PlbJNDr54GEXnzy2dX188TQIfRO6MOFv/X6KVnqn/UDN79F8kb8vNwi9YyL5IOFi91qDqfTyfPDh9KtVzwXL1wAoETJknSMiP7HZ292i5RNbFQ4a77chK+vr8flv5O3SNnERHZk3YbNV5X/8cf+Wf6o6H+W/2a3SOCd9nMnbpFuSm4vUrFixW64nd1uZ+jgh5n42tXelmWLv6BXfB9s+fC2CASXzPoM5QtXUilaOMD9njk1nbSMLPfrVg2rsP/47/j4GHA6NTSnRlCA0dPDlYZsr83t8Ob8df4cha7x/qxYtphuveJJs6Xliu89L5U3y38nKIhXKzf3shfJK6xMXMr+fXt5ZazubXllwmSaNm+he4v+p3tbHurj8hYtW5mnmIu/3Munbwzmwc4tyLI7GD97DZvefxanU+Oy2cbgsTlOnIE97uOpyV/gdGqMGRZHp7Z1GTDqIwlHmj9ke21uhzdnw/o1xMRd4/3ZmsyMebr3Z9BDCWiaxsdfeO79ke11ko0Mr1Zu7lkvkhDCD/gSaArsB0Zrmrb7etverBepoFzvFsmb3A23SAXlVrdIBSWvvUj5JS+3SAXhbrhFKmh8mbOp74gXSdO0LHSjo0Kh+H/MXT3QTqFQ3NuoBKNQKKShEoxCoZCGSjAKhUIaKsEoFAppqASjUCikcccH2jmdTvdYDG9jtaYSaBTIyqOBRiF9BKhsrYjs+LfDi4RdrvfHxyDv77A1NRWZ03WsqalStSV3vRfJ6dTcagtvY8uws+Xj/xIYJMkplJrKbxesmLw3iv0fXLmin0Cjv5zBguWCdaeTXZK3KMioz1wPNMnx/hjsfkzZ8jNOXzmTUA32dEZH1pQSO5tMhxM/u5zzm+lwYhACSaf3lnHveIIxGAxynTn+flKdMKY0gX+AnAQGkOZKvkZJ+wgM1BOMrDrKdJkvZZ4Dp68Jh6+/tPiy3Ve+WQ6p9WMQQnmRFArFvw+VYBQKhTRUglEoFNJQCUahUEhDJRiFQiENlWAUCoU07uoEszVpI/FdoojvEkXj2pVYumghfbrHkNAtx5vz3JMF8+a0admUYsE53pnOMZHExUTk2wvz1/mzPBB3Py1rlsBut/PjscM82jeOR/vG0blNfT77YA5Op5PnHu3HwF5RnPnzdwCmjP0PFnPKLeMfO7SXx/vG8OQDccx8bTT2rCyG9YkmqlEF/jx1AtAHL4584iGe6NeJc6f/AODt8S+RajF7VD/Hjx0lumNbYiPbM3zoEFJSUugeF0W32BxvzpNDBxeo/u9r2ZTQ4Gu8P7nqf2g+vTzRtYozKrKa+3XT8sG83a02AAJ45v5KjI6sRrEAfXxO/6Zl8fe7u74OsutfhnfsWmStaNcSmAY4gL2apj1/o23rN2ikbdyx55Yxu0S25ZEhw/DzdXlzjEaMRiOpFgs9buDNsaZaCL7JOJhs78wDfXqx9svNrF2zikyXd6aQ0YjJaMJsMdPnJt6lE39brxoHk5GeTkZGGv8Z+hDzP1vtXqgZ4LlH+/GfsZNJtZjZtT2Zpq3u5/D+PdzXPoKdWzcx8PHn/rGPS5evADnjYC7+fZ6g4CIYjSZefWEo/Yc+R9FixZn31qsMePIFyleqyg9HD7Lnqy00btmGowe+o+X94ezansRDjz3zj/iVitx4HExWVhZ+LsXL8KFDqFGrFuXKV9DL46p/i8VCfJ9+160fgEyb9Ybxr1f/ub0/t6p/0M/B6PXfXzUOxtcgGNiiHCWCjExJ+hWAYfdVpHiAH5OTfqVSUX/qlQnix7+sVCsewNGzFhqUDebL7//+R3wfexpTuoRJXXEu4wbjYLxR/6kWyw3HwRTUO5Ydv1zJ27/o9ykgXNO0+4GSQoj6BQp28gTFS5SkePESZGS4vDn+/qxdtZxuXvTmBPjneHMC/ANYsXwZ8R56YYwmE8FFiv7j92k2Kxf/Pk/FytUwmQLIyEgnPU33zyz6+D36PvJYnuIXK1EKo9HlsvHxxeBjILR4yWuOy5/MzHTS02z4BwSSuPB9ej00xKPjANyNG/QGV75CRd1bZNXrf9WKRHrF9/E4bk45r/H+uOpfjx/A8nzUP0C7aqF8/VuONadB2cIcP2che6xspsOJn4+BQr4GMuxOImoUJ/mnC/k+DlnIrn8Z3rFrkeVFOqdpWrrrpR39SibffOny5tzfIYJDB3RvjsViISI6ltfGj2GSt7w5ubwzZouZmNg4xo0ZydhRBfPOAOzcupn72uuriFatUQuHw8GXq5ZSqUp1qlSrwcL35/DGK3m/jfnlh2OkXL5Ileq1//Fe5eq1cNgdbF6zjAqVq1Gpag0WfzSP6RNHYk317DZp/do1tG7WkAt//0X3nr1zvDkWC9Gd4hj/8ijGjfGOVyg8IpID2d4fi8v7M2YkYzyofx8BtUoG8v35nPkbbaoU5ZuTV9yvz5oz8BGCVpVCOG/J4Kwlg5jaJXiwSVlMvnfXbdLtqH9veseuRWptCiEaAMU1TTt+ze+HCiH2CiH2Xrp08ZZxNm9YT1RsF/z8/Jjy9iwmvTmdrUkbCQwMpFZYXWqF1WXn9i0FLq+fnx/TZ83l7emz2LxxA0GBQYTVrUdY3Xps25JcoNhbN64hvFM39+unR7zCq1Pn8eWqpTRrdT9OzUlkXA82rF52y1jmK5eZNmEEI1+becNthr3wMqNfn8PmNcto3KItmtNJh07d2Lx2uUfljuvSlV17D1GmbDk2rl/H2zNm8+Y7M0jatIHAwEDCwuoRFlaP7VsLVj9wdf1v2riBwFz1vzWP9X9flaJ8myuZ1CkVyC8XbDiumTSTePgcH+7+k1aVivLD+VQEsPePFFpVCinwcXgT2fV/6dIlXnr+GWbPex8/Pz/emTGHt96ZSdKmDQQFBRFWty5hdeuyLZ/xpc1FEkKEArOBf1zDaZr2HvAe6M9gbhbnr/Pn8LvGm7Ny2WK69YwnLS0Ng9BzpNXqZW9OfB9saboXBiC1ALOCs7Ky+O2Xn6gVdvWd4sG9uwlr0BiH04FB6P4Z2y2Ow263M+HFYTz531cpVqLUTbc9sn83tevr8YXBgBAG0jw4joyMDIxG3QNVuHBhTP765fSyJYvo2TuBtLQ0hMH79b/EVf9pueo/r7O+Sxc2UrGiPx1rFKNcESPli5hoXC6Y+mUKU66IkV71S7H8yHkAqhcP4LdLNgxCoAGapmG8ix70yq7/vHrHgLvLiySE8AUWAi9pmnauILE2rl9DTOw13pxtyUyfq3tzBvdPAE1jwef58+b07BrH0cOH6O7yzlzrzemXoHtnliTmzTuTlZXFUwN689Pxowx/uCdPjRiHxZxC8/va/WPblYs+ZsyUGRgMBt6dNoWdWzcxZeYHN42/dcMqfjhygHlvvQrA4y+8zOKP5nF4327+PPUrDz76DPdHxgGwZumnjJgwDWEw8OGsN/h2+2Zeeft/ea6f5E0bmTNrGgDVqtUgPDIah8PBti1JzH3vQ1JSUujfV/fmfL50ZZ7jZnOt9+fVXPX/Xj7rf+mhnOY2KrIam3+6yOafLrpfZycX0J/VfLznT5xAj/qlaFCmMPN3/e7xcchCdv3L8I5di6xepAeAmcAx169GaZq263rb5rUXKT/cqhepoFyvF8nbXNuL5G1u1ovkDW7Wi+QNrteL5E3uZC+SN7hZL5K34t+sF0mWF+kL4AsZsRUKxb3D3XPDqVAo/nWoBKNQKKShEoxCoZCGSjAKhUIaKsEoFAppqASjUCikccetAk6nE2uqHC+SzZqKj0OOLgP00Y3paTZp8QEyJMe3+gqp8bNklz81FYM9/dYb5hODPV2q+8qamkqGvUBT9W4e35qKQcg7x3e9F8lgEBQ2ySmGweGLQ9P+MQ/FWzg0jSrFA6V5lwCaP6vPN7Jmyom/+6OnAfDzkXMxm5XdtmW1cQGPtqiIf0CglPBpNonSKxc+QuAjKQn4CIGvjwE/SZM4b9Vu7oIEY5DrzNHkOnn8fA1SnTnZiSU1Q0787OQoq46yvzcy6yjV4UtAoES3VpC/1PLb7U6pbdRXchu9GeoZjEKhkIZKMAqFQhoqwSgUCmmoBKNQKKShEoxCoZCGSjAKhUIad3WC+W7PbiI6tCU6vD0jXd6WLp2i6BwTmW9vUW5uhxfG2/gb/Vg+dQAbZz/GkjceppCfD88/eD/J84ax4JW++PoYEEKw5I2H2TJ/GBVLhwAw7YVuBAcaPdqXDG9Ubs6cOUPrFk0pWvgaL1J0Li/SEM/jnz93hm6RrQmrWBS73U5WVhbxcR1oUKUEJ3/TNSZOp5NhjyTQp0s4p//QV7F7ZeRzeXJT3S6+27ObyA5tiYnIaf/Xts/H77L2eS2yVrSrh77mrgP4BRis3WBHDRs11r7eve+6cXJ7W4YMcHlbquveFqPRiNHlbUm4ibfoZuNgCuqFSbVYpI+DqRClL2eYPQ6me/u6hFUtxZQFWxgxoAOn/zITH9GAni9+xAv92/Hb6Uv8duYSkS1r8tX+E7SsX5HNu3+mU+tavPPZjn/EP7lGj59Xb5En3ijAPUr7enWUHb9fQi/WbdjM2tVXe5Gyz2+ffjeOb7FYOJ+SftU4mIz0dNLT03hiUD8+WboOHx8fLv79F29Oepknn/8vlatU4+ih/ezYmkSrNu3Yv3c37TpEsi15I0Of+s9V8W1WC6VD5I2DsVgsNxwHc1X7v6Z95qX9g95GZY6DsVgslAwNvu1epB81TbvP5UUCuO7Ob0Vub4uPry/Fihd3e4v8AwJYkbiM3gXwttwOL4y3OXH6IsZC+vjIkCB/KpUpylcHdKPjlu9+oUW9itjSszAV8iXAvxDWtEwe792a+YnXXbH0psjwRt0svn9AgLv+s89vfB/P4xtNJoqE5MQVQlC85NULpJv8A8h0ObYCAgL49MP59B80LN/HIoPrtU+3F8zVPgvS/m8HsrxIWbleZgB/FCRetrdlUC5vi8Wse3PGjR3F2AJ4W3LHl+GF8Ta//HGR5mEV2LfwOZrULseJPy9itupzcVJSMwgp7M+Pp/7G18dA3+iG/PzHBX469TdP92vL2893pXCAZ7dJuZHpjQLdi5QdP/v8vjx6JGNGeid+bqrXrI3dbmfN8sVUqVqDqtVrseDdWUwY/QIWDxW7ssnd/vfv38dhV/uPuQvb57VIewYjhOgmhDgKlAQuXvOe24t08eLNvUiXLl3iheeeYe583dsybeYcpk6bqXtzgoIIC6tLWFj+vS2yvTDepn9cE5J2/0TT/tPZ8M2P+PgYCA7U/8oFBxpJSdWTzSvvbmLY5ET6RjVix4ETGIRg+dYj9IlqmO99y/RGZcefMWsu72R7kYI89yJ5wotjJvDGjPdYvXwxLdvobqpOXXuydsUSr+8rv1y6dIkXn3+GOa72OW3GHKa+M5PNm/T6qVO3LnXuovZ5LdISjKZpqzVNqwecBrpc8957mqY10zStWbFixa4fAN3b8tigh5k05Wpvy9LFX9A7oQ9pLm+LwWDI14zXvHph8htfBkIILpnTALiQYqVSmaK0bVwFgPDm1dlzNEe70apeRfb/8Cc+BoFT09CcGkEBhQpchmu9UQaDoUDeqGtZssg75zcv7P/uW+o3aoLTkeOm8qbjqSDY7XYeu077XOpqn7ejfgqKlAQjhMh9HW4G0vITZ0XiUvbt28u4MaOIjQpn97e7cDgcbElOIjomlg7hkXzy0QI+XvAhHSOiPI6f2wvTOTqcPa74W7ckEeWK/+lHC/gkn/FlsHjTQXpH1Gfj7MfoF92IeUu/4euDv5E8bxgNapRh9Y4ciebAbs35aM1eDv18luZhFZg8PJYVW4/meV9ZWVl06RTl9kZ9t2d3Tv13iqVjeCQff/QhHy34gPB81E9WVhadO0W5vUh7csWP6RRLx4hc8SPzHj8rK4tH4jvzw7EjDOrbjYP79vD0Y/3ZuS2JEU8/xuYv17i3XbJwAQkPDqROvYYc3P8db0wYQ6fOPTw+FhmsyNU+46Jz2v/WLTnt/25rn9ciqxepO5D9OP5nYKimac7rbXuzXqSCcqteJG/Ev929SN7mZr1I3uBmvUje4Hq9SN7kTvYieYM73Ysky4u0Csibik+hUPxruasH2ikUinsblWAUCoU0VIJRKBTSUAlGoVBIQyUYhUIhDZVgFAqFNO64VcDpdJJqkeNFslpTkWQscceXpfvIJrDgA29viuwRoDYvjvC9HtbUVNJskgYJoWtLrL4SvUWpqdgd1x0i5p341lR8JbbRW7WfO55gNE3DLikL2J0aWXYnPllyGkhGloODf1zBzyRP/LVyxuMA0rw/v/11CQBjqpxzYBS6d8XqlNPU0mwZ+BgEPgZJXiFJcXPztyWTVIecJJlmyyQkwBeRIWcyZFrmzePe8QRj8PGR6oTJtDsJDJLoRTI5KOQv58sP4B+g//UJkHQMRovesE0BcuRxJqHHl+ktMggh2Ytkkjpa2z/TB/9AefK+gEA/qd+Bm6GewSgUCmmoBKNQKKShEoxCoZCGSjAKhUIaKsEoFAppqASjUCikcVcnmOPHjhLdsS2xke0ZPnQIKSkpdI+LoltsjhfmyaGD8+2FWfLFp8R3jaFn50j++P0U8V1j6N0lGosr9jNPDPE49g+H9/FC/86MGNCN9954GYCE1tUZOagnIwf1xJJyGafTyYSnH+HFh7vw1xl9PfS5k0Zi9XCx6TSbjYH9etCnWxRDHorHbE6hX49O9Ose4z6G/wx/1KNj+Pv8Wfp3bUeb2qXcC0m/Nf4lhj3QmQkjhuNwOHA6nbww9AGGJMRw9rS+ROcb414kNY9OofPnztAtojV1KuTyFsV2oH7lEpw8cbW3KKFzLm/Rf/PmLTp/7gxdwltTq3yI+xjenf0OCZ3Dee7xgWRlZeF0Onns4QTi4zry5x+nAHh5xHOY7wIv0vlzZ+kVdR8NKodit9u5fPEC/bqG079nDE8MSCA9LQ2n08mTA/vwQLcId/1MGPW8R16n30+dJKxqObrHRpDQPRaL2UyvLtH07Bzlbj9PDcv/9wskj4MRQvwH6KVpWtv8fL5GzVps2roTgOFDh7Dg/XfpP2AwANu2JGE0GgmPjMbHx8fj2GfPnGbXzq9YtmYjAGtXLefBRwYCsH1rEkaTiY4RUR7HLlm2AlM+SKSQ0cRb/32Ckz8dp3KNOry+YIV7m5+PHaJWgybUb96GnZvX0uS+DpQsW57AwsEe7Wtb8iYaNW3Ocy+NYdbbr7Nx3Wr69deP4attyRhNJtqHe3YMwSFFmbtwNS89/hAAxw7tJysri3e/WMfC92exc8sGSpYuR72GzWjSqi3JX66i1f0RlClXgaDgInnaR0hIKJ8mrueJgf0AXckx/+PFvDHxZfc2x44cpGGT5rS8rx1frlnB/R0jKVu+IoXzsI+QkFA+W76eYQN0pcfFC3/z7c4dLF23hfkzp7Jp/WoqVq5CoybNadXmfr5cs4J2HaMoV6ECwXk8BpmEhBRlwdJ1PDVY9x0FhxTl81VJGAwGZr/9GtuSvqR8xSo0aNycFvfdz8Z1K2jbIZKy5SvkqX5y0z48gnnvfwLAmlXLeeiRQQBs25qE0WjK9/crG5lWASOQ/yXs0VeZz6aQ0Uj5ChXJSE/HZrXi7+/PqhWJ9Irvk6/YW5M343A6iO8aw+iXnsNoMpGenoHNasM/IIA1KxPp3svz2KHFS1LImONyMvj48MeJnxkxoBsLpk1E0zSM/v5kZmSQYbNh8g9g7Rcf0qXfII/3ValKVbcIzWxO4czpP67y5qxblUjXngkexTQaTQQXCXG/Pv3HSarXqgtAzToNOLz/O0z+/mRkppNus+HvH8jST/5HwsOP5n0fefAW+fsHkJGeTlq2t+iD+Tw8OG/eomvjH9q/l5ZtdEVXm/bhHNi3R4+fkY7NZiMgIJBPPpjPI4Mfz/MxyOTa8vv4+GAw6F9Vp8NBpSrV8Pf3JzPDVT/+gXz24bs8mA+v084d2+kS3YH5s6fj7/Je6d+vAFavWEaP3vn7fmUj8xbpUeDjggZZv3YNrZs15MLff9G9Z+8cL5LFQnSnOMa/PIpxYzz3wlz46zxZmZksW7MRf39/rKmpHDqwj6NHdOdMZHQsk14ZzYSXR+bLOfPbj8dIuXyJitVq8b91u3jjo1WkmlPYvW0jFavWxOmws239cspVqkr5KtVZ+em7zJ8yGltq3udlValanQP7viPivsYcPriPx59+gcMH9nP8yGEsFjPh0bFMmTCWyeNH59ubU6lqdQ7s+RqAvbt2YDFfoUr1WjjsDjasXkrFKtWoVK0Gn384l6mvjiDVS06h6jVr43DYWb18MZWr1aBajVp8+O4sXs2Ht8hsvkJh19Vh4cJFMF+5ose321mduJgq1WpQrXpNPpg/i/Gj/nPXeZEADh/YS++Ytny7czvlK1ammsvrtHbFEipXq07V6jX5+N3ZTBr7Yp7PQanSZfj2wHFWrk9i+9YtlC5ThkMH9nHkyCEsFjNRneKYOG40r47N33cA5FkF/ID2mqZtucH7bi/SpYsXbhorrktXdu09RJmy5di4fh1vz5jNm+/MIGnTBgIDAwkLq0dYWD22e+iFKRxchNZt2gHQtl1HTp74lTfemcVrb01nS9JGAoOCqB1Wl9phdflq23UP44ZYUi4z77XRPDthmr6vIkURQtA6PJZTv/wAwIBnx/D8pBlsW7+c+s3b4HRqtI3qyvb1K24W+iqWLVpI+/Aokr85QHhULGtWLGXy1JlMeGMa25I2EhgYRK3aYdSqHcbXO7Z6dAzZ1AprQNWadXj8wS5YU82EFi8BwPCXxvHKW3PZuHoZTVu1xel0Eh7bnU1rEvO1n+vx4pgJvDnzPdYsX0zL++5HczqJ7dqTNcs98xYFB4e4k0aqxUxwEf024qWxE3hr1nusSlxMqzbt0DQnsV17eRz/dtCgcTMSN+4kMq4riV/otzT/Gf0qU6a/y9oVS2hxn+51iuncg7UrluYpptFoJDAwEF9fX6Jj4/jx++O8OW02r0+dQfKmDQQG5nwHdnj4HchG1hXMw8DnN3oztxcptFjxGwbJ9hSDviq9yV+/9Vi2ZBE9eyeQlpaGMBgQ+XDZNG/ZiuPHjgBw9MghKlauDMCKZYvo3ivhaueMBzOCHXY7U0cOZ/AL4wgtXpJ0m9X9kOz4gT2UqVDZve3xg99Ro24j3cljEAiDgbS0vB+HpmmEuNSroaHF3A/mViUupmvPBK95cx575r/M/3wtRUJCadsxxv37Q/t2U6d+Y5wOp74fYcBm8+7s6X17vqV+wyY4nA5E9j48PNcNGjdlzzf6s7ydO7bQqGmLXPF30aCRK74wYDAIj8737SAzM9P9c1BQMCaTv/v1/u++pV6DJjgcTrfXKa/nIPcqBnu+/YbKVaoCsHzpInr07kNaWv6+A7m54UNeIcRE4LpTbDVNG3eLuLWARkKIx4G6QoinNU2b5WnhkjdtZM4s/SqgWrUahEdG43A42LYlibnvfUhKSgr9+/ZC0zQ+X7rSo9j1GjTCZPKnZ+dIQkOLMWz4szgcDrZvTWbmvA8wp6Qw8KF4NE3jky+W5znuV5tW89Oxg3w0bRIAA54dzbzXRmHyD6B0+Uo8NHyEe9tNiQt5atxUhMHA5/Pe4ruvkhnxxrw876t7fF+GD+nP8iWf4+vrx9wPFuJwOPhqWzLvzHkfszmFxx5OQNM0PliYtysLe1YWzwyK5+fvj/L0gF4Mf2kcs954BYPBhxb3tadeoxw7xarFnzB68nSEwcB7M6bw9bbNTJr+v1vuIysri8H9evD9sSMM7NuNF0eP54N5M9m7+xtOnviFoU89T1RsVwCWfLaASVNnYzAYmPnWZLYlbWT6/I9uGX9Qv+58f+wIA/p05cUxr9KidRsSOodTtnwFBg972r3t4oUfMfltPf70NyezLXkDM+YX+M6+QGRlZTH0oR78ePwIjz7QjedHvcpbE8diMBgoElKUN2e979522ecf8eqbs1wPgCezfctGps5dkKf97PpmJ69PegVjISMtW7ehafOW7u/X7Hc/xJySwiMP9EbTNBYuzvuVdW5u6EUSQrS/0Yc0Tdue5x0IsfNmvUgNGjXWduzam9dwHpFqsUidTW1NtXD4dIrU2dTVguXOpj51Vr9FvVdnU9usFqmzqW1WCyWC5c2mtlgsnL6UJm02dZo1laISZ1NbUy1ULhPquRcpdxIRQlQDygIeL46R3y5qhUJx73PLcTBCiNlAEaAN8A0QAOyQXC6FQvEvIC8PeRtomvYwcErTtP6yC6RQKP495CXB2IUQBuCKEOIRoLrkMikUin8JeUkwD7m2ewIoDjwitUQKheJfQ17mItVw/QPYC3g2YUahUPy/JS8JpqPrfwHURx8box7yKhSKW3LLBKNp2qu5XwshVnqzAE6HQ6oXKcsuzzljs6aSlW6TFh8gzVfuihoZHowczg+ayLz1RgUgzWbFIOSpRdJsVqwGOcoPyPY6ydPepNmsGCUumnAr71VeuqkH53pZBv05jNcQQkiTl/kZDBw+l4KfSY4XKSvdRvPKoQQGyVNOZK97Exjgd4st84exlD4AS9YxtHz2CwBsjvxP+b8ZAT4Okid1ITDIKCW+VWRJiZubMiEmAoP8b71hPrAW0tt+gFFOktGybh43L3vN/nZqwFFgesGKdDUGg0GqF8nPlCF1pG1gUJBUZ47DZf2TVUc+ruQu6xiyE4vVLu+vqOxzIISQGh/k1f/tin8j8nLWI13jYAAQQswAnpVXJIVC8W/hZpMdGwNNgGa5bpN8gca3o2AKheLe52ZXME7ADlwi5zYpHRgouUwKheJfws0mOx4CDgkhfgO+0jRNE0IIoC1w4nYVUKFQ3LvkpfvmFc21poPr/1fkFkmhUPxbyEuCCRBCFAL3Qt7y+mQVCsW/irwkmDeAr4QQS9BH8K6TW6Qczp45Q5uWTSkW7I/dbsdsNtM5JpK4mAj3+JBhjw7yyNtyO71Fsjl75gxtWzWjeJEAd/106RRF55jIfNePbCIblmX9uBjWj4vh5/kJ9GhVieSJsZz9+EGqusbkCAGLXuzI5gmxVCiuDzF4e3BLgv09Gwt05swZWrdoStHCOe0nLiaS2Oic9jN0yN1VP/828tJNvQ7IBOKBc8AtRwQJISoDu4HvgUxN06LzU7iioaGs3ZDEA316AbAleTMDBg12/2wymoiI8szbcju9RbIpGhrKmi8382Cf3oCuYnlk4CD3z0aTiUgP60c2SYfOkHToDABbJsWRdPA0X39/nokPNnVv07ByKHt/ucBXx8/Ro1Ulkg6d4Y8LqZjTPBv0FhoayvqNSfRLcLWfpM0MGDjY/fPdWD//Nm54BSOE6COE+BRYBJQHKmma1l3TtNF5jL1Z07QO+U0uACaTiaJFc/wwAS5vi9VqJcA/gBXLlxGf0NejmLfTWySba+vHPyCnfvwDAliRuIzeHtbP7aJyySD+TkknNd3O3ylXD5VPy3Rg9PMhwOiLNd3OsJjavLfxR4/3cd36yUjHasupn/g+d2f9/Fu42S3SZPQrl9c0TXsPyLjJttejoxDiKyHE8/ku3bUBIyLZv28vhw8dxGwxExMbx7gxIxk7aoTH3pbb4S263XQMj8zxRpnNdIqNY9zYUYwd7bk3SjbdWlRizXe/X/e9H0+n4Osj6NO2Kr+cNfPTmRSGdw7jzYEtKOzhbVJuwnO1n+z6eXn0SMaM9Lz9KPLGDROMpmk1gBlAVyHEaqC2EKKj60HvrTgL1ESfiR0phGiQ+82rvUgX81xYPz8/ps+ay9vTZ7F54waCAoMIq1uPsLr12LYl716k2+Utut34+fkxbeYcpk6byaaNGwgMCiIsrC5hYXXZ5qE3SjaxTcuzfu8fN3z/1UUHeGLe1/RpW4Wvjp3DIGDV7lPEt6mS7336+fkxY9Zc3pk+K6d+XO1nqwftR5F3bvqQV9O0w5qmjdc0rRsQCTQDNtwqqKZpGZqmWTVNswNrgXrXvJ/Li1TM40IvXfwFveP7YMvlbUnNo7fldnqL7hRLF39B74Q+XvMieZuSRUxk2p1cSr35RXHLmiXY/+tFfAwCpwZOp0aQqeBzmpYsurvr599Enqcxa5p2QtO0tzRN63irbYUQuWdWtQF+zU/hsrKy6NIpiqOHD9G9cye+27Mbh8PBluQkojvF0jE8ko8/+pCPFnxAeERUnmLm9haNHNST3346zvMPxDBiQDcunD9Dm6iu7m03JS4kuueDVK1djx+P7OfDdybQJrJzfg5FCllZWXSNjebokUP06BJ7df3ExNIhPJJPPlrAxws+pGMe6+d20Ll5Rdblunr5+Ln2hDcoy7vD29K5WQX37weE1+CTrT9z+ORlmlcvzsSHmrJq96k87ycrK4vOnaI4cvgQ3Tp3Yk+u+onpFEvHiFztJ/LuqZ9/Ezf0IhUoqBBxwET05zY7NU0bcaNtGzZqrH2zZ7/XywC6F2nXiYvSZlNnpllpU7241JmqV66kAPJmU6fZ9CsyWcdQecAHgLzZ1IG+dg7PSpDqLZI5m9riWgvpXo5fMjTYcy9SQdA0bT2wXkZshUJx7yB3uTSFQvH/GpVgFAqFNFSCUSgU0lAJRqFQSEMlGIVCIQ2VYBQKhTRUglEoFNKQ55LII06nU554LVWuGC0r3SZ9iLk1j1Mg8kt6mlxxXICP3LVWAnwcUs+BNTUVIVHsJr393OH4dzzByKZxhRACA+Uswme1FkJzamhO74+Gzmbd9+cA8DHKEXN1qloEyPEveZstk/WpF7LEbtbUVB6Y8w3pFJIS30QmXwy/T0rsbJxODaekNuR0ahgM8hLkrbjjCUa2eM3p1KTG9zEIufFdicXHFCAlfnbyvVfFbgDpFCJNUoIB+WI32W3UYJAvjrvhvu/IXhUKxf8LVIJRKBTSUAlGoVBIQyUYhUIhDZVgFAqFNO7qBCPDi5Sb7/bsJrJDW2Ii2jPypf9gNpvpGhtFl045XqHHH8t//LNnznBfy6aEBl/j5clV/qEelv/0rz/y1tDeTH0igU8mvYTl8kXeGtqbd57sy9wRj5KZkY7T6WTeiMd4a1g8F8/+CcAXU18mLdUzr9N3e3YT0aEt0eE59XO3e5cAujcpy4JHm/HJ0OaUDDYyuF1lPnu8BW/1rY+vQSAEzHmkMZ8/3oKyIbph4uXudQgy3vFO1auQ3f5vB1JWtAMQQjwCDAB8gIc0TTt9ve1utqJdeno6aWlpPNCnF2u/3MzaNavIzNDXcS1kNGIymjBbzPTp+8B1P59qsdy0C/D8uXMUCQnBZDIxZGB/qteoSbVq1QEwGo0YTSYsZjMJN4l/s27q65U/w1V+Yx7KD7Dg6++BnG5qhz0LH199Zf1PJr3E/T0fpFKdhhgMBtZ+MIMyVapTvGxFju/eQc0mrfjtyH7qtGzHsV1bie7/+D/i96ihaz2udwxX1c8AV/1Uz3v9gPwV8ywWC9FvbHN3U5cMNvJMVHXGJh4DoGigH68n1GfYR/t5tH0V/rhk489LabStUYw9Jy7TqFIRdv50kXa1ivPBjpP/iO9PJpv+20Fq+W/URgva/kFvozK7qW+1op2UKxghRDmgvaZpES430nWTy62Q4UXKTanSpTGZ9L9gvr6+FCtWnIyMdGwub87K5QXzCv3Dy+Mqv81qxd8/gOX5KH92cgHwLVSI0NLlMRj006g5HZQsX4VCJn/smRlkptko5B/A9sRPaN/7EY/Ln7t+fHx9KVa8+F3vXWpbszg+BsGCR5sxtlttGpQvwp4TlwDY9ctFGlUMIS3TQSE/H/wL+WDLdPBQ64p8vuvGhoM7hez2fzuQdYsUA/gIIZKFELOEEF5R53nTi5Sbo0cOc/HiBQYNeYz9+/dx2OUViukUxytjR/Gyl7xC4RGRHMj28lhcXp4xIxnjYfkPfbWZCQ/FYLl8kaAiIZw8fpApg7vx475dFC9bnjKVq+NwOPhu02pKVqhCqUrVSF70AYvfGU+a1fNpGbnr5273LhUPKoSfj4FB7+8lLdNJYX8/UjP0slnS7QT7+3Libyu+BkGXRmU4ecHGib+tDLi/EmO61ibQePdaHmW1f5nISjClgEKapkUANqC7N4J6y4uUm0uXLvHi888wZ977uldoxhymvjOTzZt0b06dunWpU9c7XqHc5d+0cQOBgfnz8jS8P4pxn20kpEQpjny9hcphjRj14WoatYvmm7VLAejx+Es8MvYtvtu8ipqNW6FpGk06xrJ38xqPynzp0iVeeO4Z5s5//57wLlnS7XznumLZ/etFyhX1dz9bCTL6YEnTv4jTNv7M6GVH6dKoDHtOXMIgBBuPnqdLwzJ3rOy3Qkb7l42sBJMCbHf9vAWok/vN/IrXsimIFyk3drudxwY/zMTX3qBU6dJXxe8VL8+bs8RV/rRc5c/rpMaszByXkCkgCGHIOYWmwCD8XFpcgF+P7KNS7QY4nQ6E0L1OGba8e53sdjuPDXqYSVP+WT93q1fowKkr1CyjP2+oXTaYcynpNK8aCkDr6sU4+PsV97aNK4Zw9M8Ul3dJnw8UcBdfwWTjrfZ/O5D12Pwb4DHXz42A33K/6VLRvgf6Q94bBcnKyqJn1zi3F2n8xMk0adqMLclJvPv+AlJSUuiX0BNN01iSuMrjQq5IXMr+fXt5ZewoAF6ZMJlmzVuwdUsS8/+nx3+wTy80TWPRspUex8/KyqJH1zi3l+fVXOV/L5/lP/7tdpIX6SqQEhUqU6RYCd55si/CYCAgOIRB495xb/v16sU89N/JCGFg3QfTObZrG4NfnZHn8q9IXMq+fXsZN0avn/ET9frJXf8PJOj1szhxZd4rRiI/nLWQkeXgk6HNuWzN5MVFhylR2Mhnj7fg7JV0Pvk6x6sU36Icryw/jlPTeCqyGu1rFefFRYfvYOmvRnb7vx3I7EWaim6CvAA8qGla5vW2k+1FkjmR7Fa9SN7g2l4kb3OzXiRvcLt7kbzNnexF8gZ3uhdJWse/pmkvyoqtUCjuDe7qgXYKheLeRiUYhUIhDZVgFAqFNFSCUSgU0lAJRqFQSEMlGIVCIQ2VYBQKhTTu+AIYsr1ITkkDCUF3FvlIdOYAODLSpMa3Wv1uvVEBkO1dsqamYuK6Yzi9golM6d4lWcqS7PgytSV3vRfJ7nDylzldSuw0WwZBRh987XIW5Mm0O3hn+wkcvqZbb5xPRrQtB0CAJLfT5RR94SKHj5zZuD5O3bcky7vkcDqZP6iZtPqx3YY5Pua0LLKEnCSZlpYFAjLIuPXG+Ylvu3m573iCMRh8CAiUN9Q+wORLYJC8+A5fEw5fOVI0yEksso4hw+6UGt/H7prJLNMr5GOXeo4DAwtJ9QplaBkESCw/Qt75vRXqGYxCoZCGSjAKhUIaKsEoFAppqASjUCikoRKMQqGQxl2XYM6fO0O3yNaEVSyK3W4nKyuL+LgONKhSgpO//QroY2eGPZJAny7hnP7jdwBeGfkcFnOKR/v6/dRJwqqVp0dcJAnd47CYzfTuGkOvLtFYXN6Zpx8fki/vTKfaxRkbpSs+5ifUY1RkNUZFViOwkA8CeK5dZcZGV6dYoD4O5ZHm5fD38+x0/H7qJGFVy9E9NoKE7rFYzGZ6dYmmZ+cod/mfGjY4396cNJuNR/p0J75LFIMf7I05JYW+3WPo0y3GHf/5Jx8tkJdKpndpa9JG4rtEEd8lisa1K7F00UL6dI8hIVf5nytA+WVjt9sZ/ujD9O0ezeTxo7GYzfTr0Yl+3XPK/5/h+S9/ms3GgL49SOgaxeCH4jGbU+jXoxN9c8V/vgDxQVI3tRCiEzDS9bIW8ISmaSvz8tmQkFA+XbaeJwb10wvo68v8jxbz5qSX3dscP3KQhk2a06pNO75cu4J2HSIpV74ihYOLeFzW9h0jmPf+xwCsWbWcBx8eCMC2rUmYjCY6RkTh4+PZOq2+BkHFojld139eSWdK0q/u15VD/fn1oo3vz6fSvEIIR85auGDNJC3L87Ei7cMjmPf+J+7yP/TIIHf5jUYT4ZHRHpc/m63Jm2jcrAXPjxjDzKmvs3H9avr11+Pv2JaMyWikfbjn9ZNNxYqVWLchye1dmjNzOo8MHOTa92aMJhORUfkvf8fIGDpGxgDQJbItmqbxQK7yG41GOhSg/LLZsG4VderW56nnP2Xcf5/n/fkz6dd/IABfbUvGaDIVqP63JW+icdPmPDdiDDPffp2N61bT14vxQdIVjKZpG1w+pA7A70BSXj9rNJkoEpLjghFCULxkqau2MfkHkOnyFwUEBPDph/PpP2hYvsr69Vfb6RrTkfmzZxDgH0BGRgY2m40A/wBWr0ykR+8+HsdsXz2Ur05cdr8uW8TImKhq9Gmkr1ifaXfi5yMw+hrIcDiJqlWMpB89X/wcYOeO7XSJ7sD82dP/4V1avWJZvsqfTeXKVd2ir5SUK5z+848cb5S/P2tXLadbr4R8x79d3qVTJ09QvERJihcv4dXyy+b3k79Rp259AMLqN6B48ZJXebvWrUqka8/8l79SlapkuBaRN6ek6Oc3PZ00V/tZuyqRbgWID5JvkYQQVYHzmqZ5dThk9Zq1sdvtrFm+mCpVa1C1ei0WvDuLCaNfwGLJux61VOky7Np/jBXrNrNj2xZKlSnDwf37OHr4EBaLmaiYWCa+MppXXx6ZZ++Mj4DaJYP4/nzOIb+0+gcmb/6VgEI+NC4XzBlzBgYhaF25KOfMGZxNyaBTneL0b1oWk2/eT0mp0mX49sBxVq5PYvvWLZQuU4ZDB/Zx5Iir/J3imDhuNK+OzXv5c1OlWnUO7NtDeOtGHD64nyeeeYHDB/Zx/MghUi0WIqJjeW38GCaNG+U1L5UM79KXa1YS26U793eI4NCBfRw7cgiLF8svi2rVa/LtN18BsOur7Vy6dIHDB/Zz/MhhLBYz4dGxTJkwlsnjR+fv/FatzsF93xHRujGHD+7Tz+/B/Rw7qsePiIplyqtjmfxK/uKD/GcwvYAVMgK/OGYCb8x4j9XLF9Oyzf04NSeduvZk7YoleY5hNBoJDAzE19eXqE5x/Pj9cd6cNospU6eTvHkjgYFB1K5Tl9p16rJj25Y8xWxTJZRdJy9f9Ttrpn4Pu//PFMq7XMjLDp3j/W//oHXlEL7/KxWB4Ls/UmhdOSRf5Y+OzS7/bF6fOoPkTbp3qXZYXWqH5b38uVn6xae0D49iy66DRER3Ys2Kpbz29iwmvjmdrUkbCQgMdMffud3z+HB7vEubN6wnKrYLfn5+THl7FpNc5Q8MDKRWWF1qFaD8Mons1JmMtDT69ehEIaOR0mXKMnnqTCa8MY1tSXr7rFU7jFq1w/h6x1aP4y9dtJD2HaNI3nWA8KhY/fxOncnEN6axNdkVv04YteqEsXO75/FBfoLpCqy+9pe5vUiXL+Xv1gBg/3ffUr9RE5wOBwaR7RfKu/cn9yTLPd9+Q+Uq1QBYvnQRPXolXOWdyau3qHSwkYgaxXmxYxXKFTESXas42fMha5QI5K/UnLkb1YsH8NulNAxCoKGhaWD0zfv97j/LXzWn/L3z513KjaZphBTVnUKhxYpjdj1EX7lsMV17xpOWloYQBoTwrN6zuR3epb/On8OvkB+hocXcv1u5bDHdXOU3CAOGfJZfNj4+Pkx4YxqLVm7A4ONDuw6RAKxKXEzXngkFrh/9/OqPI0KLFXM/2F2VuJhuPRKuaj/5nZMlbS6SEKI0kKlp2j8ySG4vUt36ja6aSpqVlcWQB3rww7EjDOrbjRdGj+eD+TPZt/sbTp74hceGP09UbFcAlixcwMSpszEYDMx4azLbkjcybd5HeS7jt9/s5PXJ4ylUyEjL1vfRtHkLHA4H27cmM2v+B5hTUhjwYDyapvHpouV5irnk4Fn3z2OjqvPDX6m82qkGGXYnf6dmsvzwOff7HaoXY8HuP3ACPeuXomHZYObl8vbcil3f7OT1Sa9gLGSkZes2NG3eEofDwbYtScx+90PMKSk88kBvNE1j4WLPLyR7JPTjycEPkbj4c/z8fJn74Wc4HA6+2pbMtLnvY05JYUj/BDRNY8HniR7Hvx3epY3r1xDjai8ADoeDHduSme4q/+D+CZDP8svm3JnTPPP4QAzCQO++D1GmXHl3/b8z533M5hQee1iv/w8Wel7+HvF9eXJwfxKXfI6frx9zPlzorp9prviPus7vh5/lr35kepGGAX6aps2+2XZ16zfSVifvklIGm9VCYYmTHa2pFl7d9JPUyY4TIyoC8iarXbqSIjW+j11fbkKm98eSLm+yozXVQojEyY4Wi4VLFnmTHW2pFqmTHa2pFmpXLHFHvEjvyoqtUCjuDe66gXYKheLfg0owCoVCGirBKBQKaagEo1AopKESjEKhkIZKMAqFQhoqwSgUCmnccauA0+nAZpXjRUqzWfFxyJuKb7Om4mOXo1zJvY97Ob6PQ279WK2p2NLlTVS0WVPx0wpJi29NTb2l+qMgpNmsIFHdlWa7+RSLO55gshwav1+WIxfLTE8nrExhfH3kXKj5+BgY1roS/gGBUuIDCKc+nV6W383PR1z1v7fJnoQryy3m1KCQr4FCHsxC9wS7pLi5mfftKZyS3FoGezojOlYj0F+OYM/XefMUcscTjDAYMPrL+4IGBgZJdfJkiUypXidfp94wgiQN9Xa4vvmyhpLbXZM3pZ4Dh1Oq9yfA6CvVi+SU7NaS/R24GeoZjEKhkIZKMAqFQhoqwSgUCmmoBKNQKKShEoxCoZDGXZdgvj+0j2cfjOP5/l2Y9/pY9++/2rSGB8MbAroXadzwh3n2oTjOn/4DgJkTR2D1YMFvgOPHjhLdsS2xke0ZPnQIKSkpdI+LoltsjpfnyaGeeYXOnztDt4jW1KmQy+sU24H6lUtw8sTVXqeEzrm8Tv/13OuUzdxZ04iLao/ZbKZH52i6x0W5yz+8AF4kgCWff0rvrjH0jIvkj99P/cMb9Uw+vVEAe/fsJqpDWzpFtGeUy4vUNTaKLp1y6v+Jx/LvRZLtjZJJdK3ijIqo6n7dtHwwU7vVAvRhLU+3rcSoiKoUC9B7Gfs3LeuxV0u2lwrkeZECgKVAIJAC9NE0LSMvny1VtjxvLVhOIaOJKS89zm8/HadKzTB2bFpDidLlAPjl+yPUbtCEhi3asGPTGpq17UipshUILBzsUTlr1KzFpq07ARg+dAgL3n+X/gMGA7BtSxJGo9Fjr1BISCifJq7niYG5vE4fL+aNiTlep2Mur1PL+9rx5ZoV3N8xkrL59DplZGRw9Mhhd5n7Dxjk/tlkMhEekX+v0Nkzp/nm669IXLMRgLW5vFHbXd6lDvnwRmVToWIl1ri8SI8O7M/cWdN5ZECOF8lkMhFRAK8TyPVGycLXIKgQcvW4mGYVinDJlgVAxaImTlyy8cN5K80qFOHoufx5tWR7qUDeFUwnYLfLi7TH9TpPhJYoRSFjtivHB4PBh93bN9G0dXuEQR8MZjT5k5mZQXqaDZN/AKs++4BuDwz2uJB+fjmDjwoZjZSvUJEMt1fIn1UrEukV75lXKC9eJ3//AN0/k+11+mA+Dw/On9fp048+oN+DjwAQEBDgLn9AQACrViyjp4flz83W5M04HQ56d41h1IvPYTSa3N4of/8A1uTTG5VNbi+Sr68vxYoVJz0jHaurXlYuX0avAnqRZHqjZNGualG+/i3HTNGgTGGOnUsle3XbTIeGn8Hl1bI7Ca9RjC0/e754/u3wUslKML8CRtfPIYDHR3/ix2OkXL5Epeq12LRyMRFdcwRQlarVxGG3s2VtIuUrV6Ni1eos/2Q+cyaPwprq2bSD9WvX0LpZQy78/Rfde/bO8fJYLER3imP8y6MYN6ZgXp5rqV6zNg6HndXLF1O5Wg2q1ajFh+/O4lUPvU5ZWVl8s/Mr2nXoCOiWytxeoeiYOCaMG834fHqR/v7rPJmZmSSu2UhAgD9WayqH9u/jmMsbFRkTy6RXRjPBA2/U9cj2Ig3M5UUym81Ed4rjlbGjeDmfXiTZ3igZ+AioVTKQH/7KGYLfpkoIu05dcb8+a87AYBC0qlSEc5YMzpkziK5VnAeblPHIq5WNTC+VrATzM9BSCHEMaAZ848mHzVcuM3vSSF6YOJ0D335F3cbN8St09XyQIc+P5aXXZrFlXSINmrfB6dS4P6YrW9flbfX/bOK6dGXX3kOUKVuOjevX8faM2bz5zgySNm0gMDCQsLB6hIXVY3sBvDzX48UxE3hz5nusWb6Ylvfdj+Z0Etu1J2uW593rtOSLhfTu08/92s/Pj6nTZ/PG2zNI2qx7hQriRQoOLsJ9bdsB0LZdR3478StvTJvFa1Ons+Uab9RX+YgPuhfppeefYfY83Yv0zow5vPXOTJI2bSAoKIiwunUJq5s/L5Jsb5QMWlcuyrencp7F1S4ZyC8XbO4R19ksP3yeD/ecplWlEH74y4oQgr1/pNCyUohH+5PtpZKVYAYAGzVNqwusA/rnfjO3Fynl8qWrPuiw23n9v0/w2IvjCS1RipM/f8+urRsZNbQPp375kQUzXnNve+zAHmrWbYTT6cRgEBiE4ZaTr3KTkZHzWKhw4cKY/F1StCWL6Nk7Qff+GAwID31LeWXfnm+p37AJDqcDYdD9PDYP9vPzzz+x4P13SejRmR++P85783SBQ+LSRfTs3QdbLm9Oaj68Oc1btuL40SMAHD1yiIqVKgO6d6l7r4QCe5fsdjtDBz/MxNeu9iItW/wFveKvLn9+vD+yvVEyKB1spGP1UJ5vX5myRUyUDzHRqFyw+3XP+jm329WKBXDyUhoGoTuONA2PrmBuh5dK1lwkAWRnjgvAVU8vc3uRaoQ1uCo1b9+4mp+OHuT9dyYA+pVKz4eHAvBc/84Mena0e9svly3kufFvIwwGPp3zJnt2JDPqrfl5LmTypo3MmTUNgGrVahAeGe32Cs1970NSUlLo31f38ny+dGWeYmZlZTG4Xw++P3aEgX278eLo8XwwbyZ7XV6noU/l8jp9toBJLq/TzLcmsy1pI9Pnf5Tn8o+fOMX9c1xUe4Y+8ZS7/HNcXqSH+7m8SEs89yLVa9AIk78/PeMiCS1WjGHDn9W9OVuTmZnLG4Wm8UkevVG5WZm4lP379vLKWN2L9MqEyTRt3oKtW5KY9z/di/RQH73+v1i20uP4sr1RMlh2KMebNSqiKkk/XSTpp4vu1yuOnHe/365aUT757jROoHu9UjQoW5h3d/2R533dDi+VFC+SECIEWIz+HCYL6Ktp2qXrbVsjrIE2L1HO5WlGmpWG5YtIdfJcscme7KgvdyBrsmOKqztS2mTHdBsg14skc7KjNdUidbKjxWJh9LrvpU129LGnMbFTLan1X6ZEyO31ImmadgWIkRFboVDcO9x1A+0UCsW/B5VgFAqFNFSCUSgU0lAJRqFQSEMlGIVCIQ2VYBQKhTRUglEoFNK441YBzekkI837w/ABMtNtWK3ypuJbramkpWVJiw/g68zTKhf5RrYXyZ4hR0mTjdWait3h2TIFnmCzpqJlyfuaWFNTMUh0axns6VKnQdwq9h1PMEZfAzVLBkmJbbPCnxdtGG1ynD8ZaTbKhfoTIMk5A7ilXAZJYiQf1xIY2f97m+ylimS5vwRg9PXB5CfnD4nDV/5aMU+0rkSAJLeWzWbl+eVHyDIYb71xPvC7xR/AO55gDAaDVKeN0e6LKUBOAgMICAyQNowfchKLrH04XVNFZMXPMOh34TK9PAaDkB5fphcpkwyp34Esg5FMgxyx261Qz2AUCoU0VIJRKBTSUAlGoVBIQyUYhUIhDZVgFAqFNO7qBLM1aSPxXaKI7xJF49qVWLpoIX26x5DQLcbttXnuyUc98rb8ff4s/bu2o03tUtjtdpxOJy8/P5Sh/eJ4sn93rly6iNPp5IWhDzAkIYazp3Vv0RvjXiQ1H94imc6i48eOEh3eltio9gwf5h2v0+08hu/27CayQ1tiInK8PNd6kR4vgBfp7JkztGnZlGLB/tjtdsxmM51jIomLifCa90cmaTYbj/TtQXzXKAY/FI/ZnELfHp3o0z2n/T8/3LP2D1CqsJFlQ5ryds8w3uheh9LBRqb3rsu0XnUZHV0Dg9C7/yd0rsWM3nUpWVhfD/uZ9lUILORZt72sFe18gYVAKeA7TdNG3Gjb+g0aaRt37LllzC6RbXlkyDD8fPUxJ4WMRoxGI6kWCz3ir69WsKZaOJ+SflU3dUZGOhnp6bz0+EPM+XQVv/xwlCWf/I9xb87hy5VLuHL5Io2atebbHck0adWWIwf20Or+CL7ZtplHhj17Vfx0WyoVi9+4mzojI4Pnn36ck7+d4PHhz5KZqY8ZKFTIiMlkwmI2X7Vo9/XIHgh3vX1kZWW51SvDhw2hRs1alCtfAdAXvDaajFjMFuJvsg+zy2Ig6xgyXGskX68b+fy5cxQJCdG9PAP7U71GTapVq56r/Hr8hL4P3DB+qsVyw27q9PR00tLSeKBPL9Z+uZm1a1aR6VqHuZDRiMlowmwx0ycP8WWuaHc59frd1OvXrOTH74/x/IgxzHz7dcqULYev63y7699ipkfvG6tFrKkWnl1+5Kpu6lKFjQxuVYEpm38BIMjog6aBNdPB4FYV+P5cKhesmTSvFMLBP1OoW6Yw3526QsvKRVm8/8xV8Qs509nwXPgNV7STdQXTEzikaVpHwF8I0bAgwU6dPEHxEiUpXrwEGRnp2Gy6t2jtquV065Vw6wC5MBpNBBcJcb8uUaoswjXWxGJOoUjRUEz+/mRkppNus+HvH8jST/5HwsOPelxumc4iuInXyVU/q5Z77nW6ncdwPS+S+/y6vEgF8fKYTCaKFs1xVAW4vEhWq5UA/wBWLF9GfAG9PzKpXKWqO6GnpKRw+s8/cnm7Ali7KpFuPT1r/9k0Kl+E6b3r0rtRGVIzHFgz9asgh6bh0DQy7E4K+Rjw9/MhLctJjwalWXX43C2i/hNZCaYqcNj180GgdUGCfblmJbFdunN/hwgOHdjHsSO6tygiOpbXxo9h0rhR+fa2hIQWIzMzk4SoFiR+/iHhMV2pUr0WDruDDauXUrFKNSpVq8HnH85l6qsjSM2jt0i2syib9euu8Tod8J7X6XYdQ24vz/79+zjsih9TQC/StXSMiGT/vr0cPnQQs8VMTGwc48aMZOyoEXeNFyk3VapW58C+7whv3ZjDB/fxxDMvcPjgfo4fPUyqxUxEVCyvvTqWSa+M9qj8l6yZDPj0AP9ZfowmFYpQtVgAAMUC/WhSvgh7f7/C75fT8DFAeM3i/Hkljd8vp9G7URmGt6tMgAejpmUlmB+B9q6fOwJFb7LtLdm8YT1RsV3w8/NjytuzmPTmdLYmbSQwMJBaYXWpFVaXndvzt3D4t19tIahwMEs372HoMyP59H+zABj+0jheeWsuG1cvo2mrtjidTsJju7NpTWKe4sp2FmUT1zmX1+nLdbw9fTZvvu0dr9PtOIZLly7x4vPPMMflRZo2Yw5T35nJ5k16/Dp161Inn16ka/Hz82P6rLm8PX0WmzduICgwiLC69QirW49tW7zrvfIGSxctpH3HKLbsOkBEVCxrVizltakzmfjGNLYmbyQgMIjadcKoXSeMndu35jlullMj3e7EqcG3v12mcrEA/AyC/0ZW550tJ8hWMH2w6w/eSv6ViJrFOXTajEEIdvxykfCaxfK8L1kJZg36rVEykAGcz/1mbi/SpUs3lz7+df4cfoX8CA3NOaiVyxbTrWc8aWlpGITuE8q3t0jTCA4JASAkNPSqK5RD+3ZTp35jnA6n7ocRBmy2vE0ck+0sgut4nUze9TrJPga73c5j1/EiLXV5kbzh5bkeSxd/Qe/4PthyeZFS7xIvUm40TSPEdYsXWqyY+8H0ysTFdO2he6lyzm/ey+/vl/O1r1emMGdS0vlPeFVWHTnPqctXT04NKx3Ej39ZMQiB0+Ve8vfgQa8sq4ADeBpACPEesOma991epPoNGt30KfPG9WuIcXmEAN3Lsy2Z6XPfx5ySwuD+CaBpLPg8b1cW9qwsnhkUz8/fH+XpAb0Y/tI4Tv7yE8Me6IzmdDLuzbnubVct/oTRk6cjDAbemzGFr7dtZtL0/+VpP7KdRQDJm/Pgdern8jotWelxfNnHsOI6XqRmLi/SfJcX6UGXF2lRPrxIWVlZ9Owax9HDh+jeuRPjJ06mSdNmV3l/+iX0RNM0liSu8ji+bHrE9+XJwf1JXPI5fr5+zP1wIQ6Hg6+2JTNtzvuYzSkM6Z+Apmks+Cxv7R+gftlgBrWqQJZD4+gZMwYBbasVo2RhI70almb5oXN8fUK3DMXVLcW0rSfQNI0BLcvTsnJRJm/8Oc/7ktWLVA74DHACn2ia9tGNts1rL1J+uF4vkje5VS+SN7hZL5I3uFUvUkG5WS+SN7hZL5I349+JXiRvcL1eJG9yq14kWVcwp4EOMmIrFIp7h7t6oJ1Cobi3UQlGoVBIQyUYhUIhDZVgFAqFNFSCUSgU0lAJRqFQSEMlGIVCIY07bhVwOp1YUy1SYtusqWSkyfMKZaRZsVnlOXkA0mxynFHZyPYiZabZpMa3WlOlKV3A5S2SpHTJjm9zqWlkYLNZb6kWKQi3ii1lJK8nCCH+Bk558JHiwAVJxbkd8W/HPlR8Ff92xq+kaVqJ671xxxOMpwgh9t5oWPK9EP927EPFV/HvlvjqGYxCoZCGSjAKhUIa92KCee8ej3879qHiq/h3Rfx77hmMQqG4d7gXr2AUCsU9gkowCo8QQuKgE0WeuJfOwT2VYIQQRW7TfqTUi+zyCyFCJcYWQoiqmqZpshq4EMIghKgqI3au+C0lxx8khCgkKb4QQpQF/G65cf73EezNePdEgnFV7EfAHCHEI0KIFhL2YRBCTAbQNM3pzS+RK/Z8YLYQoq+3G6Ar/ofAe0KIB70ZOxcvASuFEGEykowrqS8Gengzbq74AlgPtJEU3wAsBaYBTSXFXwi8AzwmhCjv7fiuNjrN1UabeCPuPZFggEcAK66FxIFOQohYL+/jI+BFIcRsAC9/iT4ELgFvANHAfV6Km80C4C9gJNBWCFHDy/EBTqAfwxwhREtX/Xiz/SQCFk3T3hFC9BRCtBFCBHoxfgtghyv+i0KIrt76Erl4H9gNdAPGCCEqeTE2wATADPQHSgNlvBx/BJAFjAaCgYeEEAVup/dKgtnv+t8JLAO+BRp4+SRO0jTNCNhcJgQ0L3SxuTS6n2maNlrTtKPACqBXQePmim8AZmuaNlLTtF/Qh3mPF0I8JYQo6a39AGuBt4ApwBtCiKZARS/G/x+66uZn9GTwJDBACFEgp1Yu/gSqCyGWo99ihAFdhRAFvqIRQgSgn+M3NU3bASQD1V3veeuP1E7gNOAPlAdGCyGeE0JEeCn+N8BJ9D8inwMV0P9YFWg18nslwfzs+tce8EGvjECgmhf38SuAy6PtTjJCiIpCCP/8BtU0zQ5sc93m+aI39KKu2LULEtsV3wnsdcVriH4lMw3dC+7N4eQmIB7d1Pky+hXHIG8F1zRtPfAxMEPTtFHAKPTz663bjTPo9eRAv+J7G/gN/ctaIDRNs2maltvcdgn9SsMrf6Rc7AGOAXOBKuhX82fQLare4Hv05PUEEAsEATUp4B+ReyLBaJqWDqwEGgFd0WeB/wV0dH1xC/xXQtM0R3YcTdOeA84IIXajn9ACOR80TcvSdOzAEeB7IUQvYDr6SS0QuRrx98AzmqbtBS4DEd66jdE07QowD90W0RNdDdxECOHjrb/SmqZtAOa4fv4dfcJdcy/F1tDb0GkgAQgFjECUN4/Bta+PgXQhxGgvxrykadoKYDaQpGnan0AR9McFfgUtv6ZpfwMfAGeBluh/PLagJ5kCBb5n/gHl0J/HfAp8CdSUsI/swYdR6LO863gzNvr97Vn0v6a1JNVTPPA1UMPLcUsBnwCvul77SjzX/YBDQDUvxy2LfnXxIXrCqS2p/TQDXgBMEs7BRPSr1O3eLn+u/YS76r9yQeLckyN5Xd2xBk3TpE1ZF0JUAXw0/bmGt2OPAFZqmvaThNhGYACwTVL84tn1LoTw0XSLp7f3YQIeAr7WNO0Hb8d37aMoehu6ubs4//GLoLefSxJiB6M/5E3XNM2TpU482UdL4HJB29A9mWDudWR9MXPFN2j6sxlpCCGEJrHx3I5jUMhHJRiFQiGNe+Ihr0KhuDdRCUahUEhDJRiFQiENlWAU10UI0UEIcUoIsU0IscrVs5PXz453fb6REGLITeLnaZBYdry87l9x96ASjOJmfKppWgf0kdPx4NlMc03TDmqa9sEN3u6A90ahKu5S7rgXSXFPcBBYJYRoBdQUQnRHH6hWCvgbfeBaYfTpA2lAJvr0iA5ApKZpY4UQT6CPz0kDHgcGAj2FEEnA2LzEk3+YCm+jrmAUeaEdkIE+8C0aeBRYrWlaOPoXP971u/c1TYvjmukPrkmXCUAbTdM6os8r+wh4QdO0FzyNp7h3UFcwipvxsGu28XFgNbDP9fs6QFMhxDD0eVpfoN/urHW9v/+aOFWA/dmDCzV9vZ3c73saT3GPoK5gFDfjU03TOmqaNhx9FnL2yNofgTc1TeugaVor9AmhvwENXe83vibOCaBx9vMb1/9Z6DPj8xNPcY+gEowiP7yH/vwkWQixBWiCvuDSMCHEl+i3U240faZuIvCNEGIrUAP9Vmi0EGKcp/EU9w5qqoBCoZCGuoJRKBTSUAlGoVBIQyUYhUIhDZVgFAqFNFSCUSgU0lAJRqFQSEMlGIVCIQ2VYBQKhTT+DzZjcdZS4dqCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_cm(model2(torch.FloatTensor(val_sp).to(device)).argmax(dim=1).detach().cpu(), torch.LongTensor(val_label), 'model2.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "aa02f7e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds1 = model1(torch.FloatTensor(val_sp).to(device))\n",
    "preds2 = model2(torch.FloatTensor(val_sp).to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c963c5f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([23403, 10])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "f574e946",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = preds1 * 0.9 + preds2 * 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "9b62c99a",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = preds.argmax(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "b1f91b27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5734307567405889"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ans.cpu().numpy() == val_label).sum()/len(val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "708e6c96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5719779515446738"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(preds1.argmax(dim=1).cpu().numpy() == val_label).sum()/len(val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46e10cca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "876712ca",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'TensorDataset' object has no attribute 'size'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-119-f8d23f3f0dd2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mboost_num\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mboost_epoch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;31m# 分类器学习, 更新训练集\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mtrain_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensorDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mtrain_loader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_last\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *tensors)\u001b[0m\n\u001b[1;32m    165\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m         \u001b[0;32massert\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Size mismatch between tensors\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    165\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m         \u001b[0;32massert\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Size mismatch between tensors\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'TensorDataset' object has no attribute 'size'"
     ]
    }
   ],
   "source": [
    "boost_epoch_num = 4\n",
    "for boost_num in range(boost_epoch_num):\n",
    "    # 分类器学习, 更新训练集\n",
    "    train_dataset = TensorDataset(train_tensor, y_train_tensor)\n",
    "    train_loader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "    model = DNN().to(device)\n",
    "    boost_feature = []\n",
    "    boost_label = []\n",
    "    optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
    "    train_best = float('inf')\n",
    "    best_model = None\n",
    "    print(f'boost round: {boost_num}/{boost_epoch_num}')\n",
    "    for epoch in range(epochs):\n",
    "        print('='*20 + f' Epoch: {epoch} '+ '='*20)\n",
    "        train(train_loader, model, optimizer, criterion=criterion, labels=train_label)\n",
    "        loss = predict(val_loader, model, criterion=criterion, labels=val_label)\n",
    "        if loss <= train_best:\n",
    "            train_best = loss\n",
    "            best_model = model\n",
    "    torch.save(best_model.state_dict(), f'./best_model{str(boost_epoch_num)}.point')\n",
    "    # 开始boosting\n",
    "    train_tensor = torch.cat(boost_label, dim=0)\n",
    "    y_train_tensor = torch.cat(boost_feature, dim=0)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
